1 #!/usr/bin/env python
2 # coding: utf-8
3
4
5 import requests
6 import pandas as pd
7 import json,random,time,datetime
8
9 # userAgent
10 userAgent = [
11 "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
12 "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
13 "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0) Gecko/16.0 Firefox/16.0",
14 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10",
15 "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
16 "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
17 "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36",
18 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17"
19 "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:2.0b13pre) Gecko/20110307 Firefox/4.0b13pre",
20 "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0",
21 "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11",
22 "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"
23 ]
24
25 # get city
26 def getCityMsg():
27 headers = {
28 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
29 "Referer": "https://flights.ctrip.com/itinerary",
30 "Content-Type": "application/json"
31 }
32 url = 'https://flights.ctrip.com/itinerary/api/poi/get'
33 r = requests.get(url=url,headers=headers).text
34 # print(len(r))
35 # get city msg
36 city = {}
37 city_load = json.loads(r).get('data')
38 for data in city_load.keys():
39 if data != '热门':
40 tmpdata = city_load.get(data)
41 for i in tmpdata:
42 # print(i) # A
43 for k in tmpdata.get(i):
44 name = k.get('data').split('|')
45 cityNumId = name[2]
46 cityId = name[3]
47 cityName = name[1].split('(')[0]
48 city[cityName] = [cityId, cityNumId]
49 return city
50
51 # 生成自今日至往后7天日期
52 def get_date():
53 dateList = [] # 存放时间list
54 formatDate = datetime.datetime.now() # 生成今日的格式化时间
55 strDate = formatDate.strftime('%Y-%m-%d') # 生成字符串日期
56 stpDate = datetime.datetime.strptime(strDate,'%Y-%m-%d') # 将字符串转为日期格式的日期
57 for i in range(7):
58 stpDate += datetime.timedelta(days=+1) # 日期叠加1
59 dateList.append(datetime.datetime.strftime(stpDate,'%Y-%m-%d')) # 放入list
60 return dateList
61
62 # get page text:routeList
63 def get_routeList(headers, load_json):
64 response = requests.post(url = "https://flights.ctrip.com/itinerary/api/12808/products",data=json.dumps(load_json), headers = headers).text
65 result = json.loads(response)["data"].get('routeList')
66 if result is not None:
67 return json.loads(response)["data"].get('routeList')
68 else:
69 print('Get 【{} --> {}】 Page is failed ！'.format(load_json.get('airportParams')[0].get('dcityname'), load_json.get('airportParams')[0].get('acityname')))
70 print('休息30s后再来……')
71 time.sleep(30)
72 get_routeList(headers, load_json)
73
74 # get Data
75 def get_data(index, df, routeList):
76 for i, route in enumerate(routeList):
77 if route.get('routeType') == 'Flight': # 只要航班
78 index += 1
79 # route is dict
80 # we need route inside legs, legs is list, but its lengths is 1
81 # so we should legs[0], legs[0] is dict
82
83 # flight
84 flight = route.get('legs')[0].get('flight') # dict
85 # cabins
86 # cabins 里面又有不同长度的list ，因此考虑将cabins单独作为一个表
87 # cabins = route.get('legs')[0].get('cabins') # list
88 # characteristic
89 # characteristic = route.get('legs')[0].get('characteristic') # dict
90
91 #### about flight
92 if flight is not None:
93 # common attr
94 df.loc[index,'airlineCode'] = flight.get('airlineCode')
95 df.loc[index,'AirlineName'] = flight.get('airlineName')
96 df.loc[index,'durationDays'] = flight.get('durationDays')
97 df.loc[index,'flightNumber'] = flight.get('flightNumber')
98 df.loc[index,'mealFlag'] = flight.get('mealFlag')
99 df.loc[index,'mealType'] = flight.get('mealType')
100 df.loc[index,'comfort'] = flight.get('comfort')
101 df.loc[index,'craftKind'] = flight.get('craftKind')
102 df.loc[index,'craftTypeCode'] = flight.get('craftTypeCode')
103 df.loc[index,'craftTypeKindDisplayName'] = flight.get('craftTypeKindDisplayName')
104 df.loc[index,'craftTypeName'] = flight.get('craftTypeName')
105 df.loc[index,'delayedTime'] = flight.get('delayedTime')
106 df.loc[index,'oilFee'] = flight.get('oilFee')
107 df.loc[index,'punctualityRate'] = flight.get('punctualityRate')
108 df.loc[index,'sharedFlightName'] = flight.get('sharedFlightName')
109 df.loc[index,'sharedFlightNumber'] = flight.get('sharedFlightNumber')
110 df.loc[index,'specialCraft'] = flight.get('specialCraft')
111 df.loc[index,'stopInfo'] = flight.get('stopInfo')
112 df.loc[index,'stopTimes'] = flight.get('stopTimes')
113 df.loc[index,'tax'] = flight.get('tax')
114 # arrival
115 df.loc[index,'arrivalairportName'] = flight.get('arrivalAirportInfo').get('airportName')
116 df.loc[index,'arrivalairportTlc'] = flight.get('arrivalAirportInfo').get('airportTlc')
117 df.loc[index,'arrivalcityName'] = flight.get('arrivalAirportInfo').get('cityName')
118 df.loc[index,'arrivalcityTlc'] = flight.get('arrivalAirportInfo').get('cityTlc')
119 df.loc[index,'arrivalTerminalName'] = flight.get('arrivalAirportInfo').get('terminal').get('name')
120 df.loc[index,'arrivalDate'] = flight.get('arrivalDate')
121 # departure
122 df.loc[index,'departureairportName'] = flight.get('departureAirportInfo').get('airportName')
123 df.loc[index,'departureairportTlc'] = flight.get('departureAirportInfo').get('airportTlc')
124 df.loc[index,'departureCityName'] = flight.get('departureAirportInfo').get('cityName')
125 df.loc[index,'departureCityTlc'] = flight.get('departureAirportInfo').get('cityTlc')
126 df.loc[index,'departureTerminalName'] = flight.get('departureAirportInfo').get('terminal').get('name')
127 df.loc[index,'departureDate'] = flight.get('departureDate')
128
129 #### characteristic : charactor
130 # characteristic:charactor
131 charactor = route.get('legs')[0].get('characteristic') # dict
132 if charactor is not None:
133 df.loc[index, 'businessAircraft'] = charactor.get('businessAircraft')
134 df.loc[index, 'discountAmount'] = charactor.get('discountAmount')
135 df.loc[index, 'discountShowType'] = charactor.get('discountShowType')
136 df.loc[index, 'flyMan'] = charactor.get('flyMan')
137 df.loc[index, 'groupTicketPrice'] = charactor.get('groupTicketPrice')
138 df.loc[index, 'hotFlight'] = charactor.get('hotFlight')
139 df.loc[index, 'hx'] = charactor.get('hx')
140 df.loc[index, 'infantSoldOut'] = charactor.get('infantSoldOut')
141 df.loc[index, 'lowPriceDiscount'] = charactor.get('lowPriceDiscount')
142 df.loc[index, 'lowestBabyCfPrice'] = charactor.get('lowestBabyCfPrice')
143 df.loc[index, 'lowestBabyPrice'] = charactor.get('lowestBabyPrice')
144 df.loc[index, 'lowestCfPrice'] = charactor.get('lowestCfPrice')
145 df.loc[index, 'lowestChildAdultCfPrice'] = charactor.get('lowestChildAdultCfPrice')
146 df.loc[index, 'lowestChildAdultPrice'] = charactor.get('lowestChildAdultPrice')
147 df.loc[index, 'lowestChildCfPrice'] = charactor.get('lowestChildCfPrice')
148 df.loc[index, 'lowestChildPrice'] = charactor.get('lowestChildPrice')
149 df.loc[index, 'lowestPrice'] = charactor.get('lowestPrice')
150 df.loc[index, 'promotion'] = charactor.get('promotion')
151 df.loc[index, 'providerHx'] = charactor.get('providerHx')
152 df.loc[index, 'roundTripDiscounts'] = charactor.get('roundTripDiscounts')
153 for i, stdPrice in enumerate(charactor.get('standardPrices')):
154 diffCabinCla = stdPrice.get('cabinClass')
155 df.loc[index, 'price' + diffCabinCla] = stdPrice.get('price')
156 df.loc[index, 'superFlyMan'] = charactor.get('superFlyMan')
157 df.loc[index, 'weight'] = charactor.get('weight')
158
159
160 #### carbins
161 # 由于cabins 里面又有不同长度的list， 会出现很多的空值列。因此考虑将cabins单独作为一个表
162 # cabins = route.get('legs')[0].get('cabins') # list
163 # for i, cabin in enumerate(cabins):
164 # carbin = cabin.get('cabinClass') + str(i)
165 # diffCabin = cabin.get('cabinClass') + str(i) # carbins is list, have more, we need diff them
166 # df.loc[index, 'compositionPrice' + diffCabin] = cabin.get('price').get('compositionPrice')
167 # df.loc[index, 'discount' + diffCabin] = cabin.get('price').get('discount')
168 # df.loc[index, 'discountAmount' + diffCabin] = cabin.get('price').get('discountAmount')
169 # df.loc[index, 'discountShowType' + diffCabin] = cabin.get('price').get('discountShowType')
170 # df.loc[index, 'favorablePrice' + diffCabin] = cabin.get('price').get('favorablePrice')
171 # df.loc[index, 'fdPrice' + diffCabin] = cabin.get('price').get('fdPrice')
172 # df.loc[index, 'originalPrice' + diffCabin] = cabin.get('price').get('originalPrice')
173 # df.loc[index, 'pcPrice' + diffCabin] = cabin.get('price').get('pcPrice')
174 # df.loc[index, 'price' + diffCabin] = cabin.get('price').get('price')
175 # df.loc[index, 'rate' + diffCabin] = cabin.get('price').get('rate')
176 # df.loc[index, 'salePrice' + diffCabin] = cabin.get('price').get('salePrice')
177 # df.loc[index, 'serviceCharge' + diffCabin] = cabin.get('price').get('serviceCharge')
178
179 # # refundEndorse
180 # df.loc[index, 'changeNote' + diffCabin] = cabin.get('refundEndorse').get('changeNote')
181 # df.loc[index, 'changeRuleFlag' + diffCabin] = cabin.get('refundEndorse').get('changeRuleFlag')
182 # df.loc[index, 'endorseNote' + diffCabin] = cabin.get('refundEndorse').get('endorseNote')
183 # df.loc[index, 'minEndorseFee' + diffCabin] = cabin.get('refundEndorse').get('minEndorseFee')
184 # df.loc[index, 'minRefundFee' + diffCabin] = cabin.get('refundEndorse').get('minRefundFee')
185 # df.loc[index, 'refundNote' + diffCabin] = cabin.get('refundEndorse').get('refundNote')
186 # df.loc[index, 'refundRuleFlag' + diffCabin] = cabin.get('refundEndorse').get('refundRuleFlag')
187 # df.loc[index, 'remark' + diffCabin] = cabin.get('refundEndorse').get('remark')
188 # df.loc[index, 'serviceLevel' + diffCabin] = cabin.get('refundEndorse').get('serviceLevel')
189 return (index, df)
190
191
192 # main function
193 def main(index, df,city):
194 # 初始化 时间
195 flightDates = get_date()
196 for flightDate in flightDates: # 起飞日期
197 print(flightDate, end= '\t')
198 # (广州，(can,32))
199 for (fromCityName, fromCityId) in city.items(): # 起飞城市
200 for (toCityName, toCityId) in city.items(): # 降落城市
201 if fromCityName != toCityName:
202 # headers
203 headers = {
204 "User-Agent": random.choice(userAgent),
205 "origin": "https://flights.ctrip.com",
206 "content-type": "application/json"
207 }
208 # 加载不同 load_json
209 load_json = {
210 "airportParams":[
211 {"dcity":fromCityId[0],"dcityname":fromCityName,"acity":toCityId[0],"acityname":toCityName,"date":flightDate,"dcityid":fromCityId[1],"acityid":toCityId[1]}
212 ],
213 "classType": "ALL",
214 "date": flightDate,
215 "flightWay": "Oneway",
216 "hasBaby": False,
217 "hasChild": False,
218 "searchIndex": 1,
219 "token": "3481e1f047cee3eb638f2435b7c1b860"
220 }
221 # routeList
222 routeList = get_routeList(headers, load_json)
223 # get_data
224 (index, df) = get_data(index, df, routeList)
225 print(index,df.shape, end='\t')
226 time.sleep(random.choice(range(5, 10)))
227 time.sleep(randomm.choice(range(5, 10)))
228 print('【{}】起飞,抓完！'.format(fromCityName))
229 time.sleep(randomm.choice(range(60, 90)))
230 print('起飞日期：{},抓完,写入文件！'.format(fromCityName))
231 df.to_csv('【{}】爬取:【{}】起飞航班.csv'.format(flightDates[0], flightDate),index=False)
232 return (index, df)
233
234
235
236 if __name__ == "__main__":
237 df = pd.DataFrame() # 存数df
238 index = 0 # 计数索引
239 # getCityMsg
240 city = getCityMsg()
241 (index, df) = main(index, df, city)
注意爬取速度和更换load_json中的token