import requests

from lxml import etree

import time

import pymysql

import json

headers={

    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',

    'Content-Type':'application/x-www-form-urlencoded',

    'Pragma':'no-cache',

    'Upgrade-Insecure-Requests':'',

    'Content-Length':'',

    'Host':'www.bjda.gov.cn'

}

headers_xiangqing={

    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',

    'Pragma':'no-cache',

    'Upgrade-Insecure-Requests':'',

    'Host':'www.bjda.gov.cn'

}

dd={

    'pageSize':''

}

temp=[]

dd['currentPage'] = ''

print(dd)

response = requests.post('http://www.bjda.gov.cn/eportal/ui?pageId=348736', headers=headers, data=dd)

selector = etree.HTML(response.text)

item_spider = list(set(selector.xpath('//tr[@class="chaxun_con"]//a/@href')))

temp.extend(item_spider)

for i in temp:

    print('http://www.bjda.gov.cn/eportal/ui?pageId=348738&'+i[1:])

    response=requests.get('http://www.bjda.gov.cn/eportal/ui?pageId=348738&'+i[1:],headers=headers_xiangqing)

    print(response.status_code)

    selector=etree.HTML(response.text)

    tr=selector.xpath('//table[@class="table_sjcx"]//tr')

    print(tr