selenium配合phantomjs实现爬虫功能,并把抓取的数据写入excel

# -*- coding: UTF-8 -*-
'''
Created on 2016年5月13日 @author: csxie
'''
import datetime
from Base import BasePage
import ExcelOperation as excel
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.expected_conditions import text_to_be_present_in_element
import unittest
class JobLog(BasePage):
url=""#目标url
def setUp(self):
BasePage.setUP(self)
def tearDown(self):
BasePage.tearDown(self) def test_querysql(self):
"""
查询到符合条件的信息并写入excel
"""
driver=self.driver
driver.get(self.url)
'''
设置参数
start:起始日期
end:截止日期
user:查询人员
'''
end=datetime.date.today()
start=end+datetime.timedelta(days=-6)
user=None
#起始日期
txtStart=driver.find_element_by_id("ctl01_txtSelectDateFrom")
txtStart.clear()
txtStart.send_keys(start)
#截止日期
txtEnd=driver.find_element_by_id("ctl01_txtSelectDateTo")
txtEnd.clear()
txtEnd.send_keys(end)
#查询人员
if(user!=None):
txtuser=driver.find_element_by_id("ctl01_txtUser")
txtuser.clear()
txtuser.send_keys(user)
#查询按钮
btnQueryInfo=driver.find_element_by_id("ctl01_btnQueryInfo")
btnQueryInfo.click()
#总页数
totalPages=int(driver.find_element_by_id("ctl01_uc_CommonPager_lb_TotalPages").text)
i=1
while(i<=totalPages):
#table的xpath
listTable=self.getTableData(".//*[@id='ctl01__wrLogList_gvlist']",colNO=9,loop=i)
#当前页数
currentPage=int(driver.find_element_by_id("ctl01_uc_CommonPager_lb_PageNum").text)
if(currentPage==1):
excel.save_to_excel_newsheet(listTable,excelPath='joblog.xls')
else:
excel.save_to_excel_oldsheet(listTable,excelPath='joblog.xls')
if(i!=totalPages):
#下一页按钮
driver.find_element_by_id("ctl01_uc_CommonPager_NextPage").click()
WebDriverWait(driver,8).until(text_to_be_present_in_element(('id','ctl01_uc_CommonPager_lb_PageNum'),str(i+1)))
i=i+1 def getTableData(self,value,colTag='td',colNO=None,loop=None):
'''
获取表格数据
colNO:存在合并行时,强制指定表格列数
'''
driver=self.driver
table=driver.find_element(by="xpath",value=value)
rows=table.find_elements(by="tag name",value="tr")#行集合
maxrowCount=len(rows)#表格行数 #如果colNO=None不为空,则用传进来的值作为列数
if(colNO==None):
cols = table.find_elements(by="tag name",value="td");
if(cols!='td'):
cols = table.find_elements(by="tag name",value="th")
maxcolCount=len(cols)/len(rows)#表格列数
else:
maxcolCount=colNO lists = [[] for i in range(maxrowCount)]
#print "列数"+str(maxcolCount)
#print "行数"+str(maxrowCount)
i=1
while i<=maxrowCount:
j=1;#初始化列
while j<=maxcolCount:
try:
if(i==1):
try:
innerText=driver.find_element(by="xpath",value=value+"/tbody/tr["+str(i)+"]/"+colTag+"["+str(j)+"]").text
except:
colTag="th"
innerText=driver.find_element(by="xpath",value=value+"/tbody/tr["+str(i)+"]/"+colTag+"["+str(j)+"]").text
else:
colTag="td"
if(j==3 or j==5):
innerText=driver.find_element(by="xpath",value=value+"/tbody/tr["+str(i)+"]/"+colTag+"["+str(j)+"]/a").get_attribute("title")
elif(j==8):
innerText=driver.find_element(by="xpath",value=value+"/tbody/tr["+str(i)+"]/"+colTag+"["+str(j)+"]").text
innerText=float(innerText.replace("h", ""))
else:
innerText=driver.find_element(by="xpath",value=value+"/tbody/tr["+str(i)+"]/"+colTag+"["+str(j)+"]").text
except:
innerText=""
#print "第"+str(i)+"行,第"+str(j)+"列"+str(innerText)
lists[i-1].append(innerText)
j+=1
i+=1
if(loop!=1):#不是第一次循环,列头不要
return lists[1:]
return lists if __name__ == "__main__":
unittest.main()
# -*- coding: UTF-8 -*-
'''
Created on 2016年5月18日 @author: csxie
'''
import xlwt
from xlrd import open_workbook
from xlutils.copy import copy
import os def save_to_excel_newsheet(listTable,excelPath=r'C:\demo.xls',sheetName='sheet1'):
if(isinstance(listTable,list)):
rowNO=len(listTable);
if(rowNO==0):
raise ValueError,u'传入的是list是空的'
colNO=len(listTable[0]); wkb = xlwt.Workbook()
sheet = wkb.add_sheet(sheetName)
for i in range(rowNO):
for j in range(colNO):
#print listTable[i][j]
#print str(i)+'行'+str(j)+'列'
sheet.write(i,j,listTable[i][j])
if(os.path.exists(excelPath)):
os.remove(excelPath)
wkb.save(excelPath)
return True
else:
raise TypeError,u'传入的类型不是list' def save_to_excel_oldsheet(listTable,excelPath=r'C:\demo.xls',sheetName='sheet1'):
if(isinstance(listTable,list)):
rowNO=len(listTable);
if(rowNO==0):
raise ValueError,u'传入的是list是空的'
colNO=len(listTable[0]);
rb = open_workbook(excelPath)
sheet_index=rb.sheet_names().index(sheetName)
old_rows=rb.sheet_by_name(sheetName).nrows#已存在的excel中数据行数 wkb = copy(rb)
sheet = wkb.get_sheet(sheet_index)
for i in range(rowNO):
for j in range(colNO):
sheet.write(i+old_rows,j,listTable[i][j])
#print str(i)+'行'+str(j)+'列'
wkb.save(excelPath)
return True
else:
raise TypeError,u'传入的类型不是list'
# -*- coding: UTF-8 -*-
from selenium import webdriver
import unittest class BasePage(unittest.TestCase):
def setUP(self):
self.driver= webdriver.PhantomJS(executable_path="phantomjs.exe")
#self.driver.set_window_size(1400, 1000)
#self.driver.implicitly_wait(1)
self.verificationErrors = [] def tearDown(self):
self.driver.quit()
#self.driver.close()
self.assertEqual([], self.verificationErrors)
上一篇:windows程序崩溃生成dump文件


下一篇:2019.01.21 bzoj1758: [Wc2010]重建计划(01分数规划+长链剖分+线段树)