一、作业①
| 序号 |
地区 |
日期 |
天气信息 |
温度 |
| 1 |
北京 |
7日（今天） |
晴间多云，北部山区有阵雨或雷阵雨转晴转多云 |
31℃/17℃ |
| 2 |
北京 |
8日（明天） |
多云转晴，北部地区有分散阵雨或雷阵雨转晴 |
34℃/20℃ |
| ... |
... |
... |
... |
... |
1.解析网页
cityCode:{"北京":"101010100","上海":"101020100","广州":"101280101","深圳":"101280601"}
url = "http://www.weather.com.cn/weather/"+self.cityCode[city]+".shtml"
2.获取网页源码getHtml()
req=urllib.request.Request(url,headers=self.headers)
data=urllib.request.urlopen(req)
data=data.read()
dammit=UnicodeDammit(data,["utf-8","gbk"])
data=dammit.unicode_markup
3.创建数据库
class WeatherDB:
def openDB(self):
self.con=sqlite3.connect("weathers.db")
self.cursor=self.con.cursor()
try:
self.cursor.execute("create table weathers (wCount varchar(16),wCity varchar(16),wDate varchar(16),wWeather varchar(64),wTemp varchar(32),constraint
pk_weather primary key (wCity,wDate))")
except:
self.cursor.execute("delete from weathers")
def closeDB(self):
self.con.commit()
self.con.close()
def insert(self,count,city,date,weather,temp):
try:
self.cursor.execute("insert into weathers (wCount,wCity,wDate,"
"wWeather,wTemp) values (?,?,?,?,?)" ,
(count,city,date,weather,temp))
except Exception as err:
print(err)
def show(self):
self.cursor.execute("select * from weathers")
rows=self.cursor.fetchall()
print("%-16s%-16s%-16s%-32s%-16s" % ("count","city","date","weather","temp"))
for row in rows:
print("%-16s%-16s%-16s%-32s%-16s" % (row[0],row[1],row[2],row[3],row[4]))
4.获取并打印所需信息
soup=BeautifulSoup(data,"lxml")
lis=soup.select("ul[class='t clearfix'] li")
for li in lis:
try:
count += 1 # 标记序号
date=li.select('h1')[0].text
weather=li.select('p[class="wea"]')[0].text
temp=li.select('p[class="tem"] span')[0].text+"/"+li.select('p[class="tem"] i')[0].text
print(tplt.format(count,city,date,weather,temp))
self.db.insert(count,city,date,weather,temp) # 插入数据库
except Exception as err:
print(err)
5.运行结果
6.心得体会
- 通过复现课本中项目，初步掌握了将爬取数据存储至数据库的方法；
- 对于BeautifulSoup的使用逐渐熟练。
二、作业②
| 序号 |
股票代码 |
股票名称 |
最新报价 |
涨跌幅 |
涨跌额 |
成交量 |
成交额 |
振幅 |
最高 |
最低 |
今开 |
昨收 |
| 1 |
688093 |
N世华 |
28.47 |
62.22% |
10.92 |
26.13万 |
7.6亿 |
22.3% |
32.0 |
28.08 |
30.2 |
17.55 |
| 2 |
... |
... |
... |
... |
|
|
|
|
|
|
|
|
1.解析网页
url = 'http://11.push2.eastmoney.com/api/qt/clist/get?cb=jQuery11240004083955569821329_1634088838117
&pn='+ str(page) +&pz=20&po=1&np=1&ut=bd1...'
jQuery1124036571660128497574_1634087610037({"rc":0,"rt":6,"svr":182481604,"lt":1,"full":1,"data":{"total":4674,
"diff":[{
"f1":2,
"f2":14.16,
"f3":20.0,
"f4":2.36,
"f5":150,
"f6":212627.0,
"f7":0.0,
"f8":0.03,
"f9":37.33,
"f10":"-",
"f11":0.0,
"f12":"605567",
"f13":1,
"f14":"N春雪",
"f15":14.16,
"f16":14.16,
"f17":14.16,
"f18":11.8,
"f20":2832000000,
"f21":708000000,
"f22":0.0,
"f23":2.64,
"f24":20.0,
"f25":20.0,
"f62":"-",
"f115":31.76,
"f128":"-",
"f140":"-",
"f141":"-",
"f136":"-",
"f152":2},
2.构造正则表达式抓取数据
name = re.findall('"f14":"(.*?)",', resp.text) # 股票名称
new_price = re.findall('"f2":(.*?),', resp.text) # 最新报价
...
3.创建数据库
try:
self.cursor.execute("create table shares (sId varchar(4),sNum varchar(8),sName varchar(8),sNew varchar(4),sUp varchar(4),sUpprice varchar(4),
sCom varchar(4),sComprice varchar(12),sF varchar(4),sMax varchar(4),sMin varchar(4),sToday varchar(4),sYes varchar(4),
constraint pk_shares primary key (sId,sNum))")
except:
self.cursor.execute("delete from shares")
4.获取并打印信息
NUM, NAME, NEW, UP, UPPRICE, COM, COMPRICE, F, MAX, MIN, TODAY, YES = ([] for x in range(12))
num = re.findall('"f12":"(.*?)",', resp.text) # 代码
NUM.extend(num)
name = re.findall('"f14":"(.*?)",', resp.text) # 股票名称
NAME.extend(name)
new_price = re.findall('"f2":(.*?),', resp.text) # 最新报价
NEW.extend(new_price)
...
global count
try:
tplt = '{:2}\t{:8}\t{:8}\t{:4}\t{:4}\t{:4}\t{:4}\t{:12}\t{:6}\t{:6}\t{:6}\t{:6}\t{:6}'
if page == 1:
print(tplt.format('序号', '股票代码', '股票名称', '最新报价', '涨跌幅', '涨跌额', '成交量', '成交额','振幅','最高', '最低', '今开', '昨收'))
for i in range(len(NUM)):
count += 1
self.db.insert(count, NUM[i], NAME[i], NEW[i], UP[i], UPPRICE[i],COM[i], COMPRICE[i], F[i], MAX[i], MIN[i], TODAY[i],YES[i])
print(tplt.format(count, NUM[i], NAME[i], NEW[i], UP[i],COM[i],COMPRICE[i], F[i], MAX[i], MIN[i], TODAY[i],YES[i]))
except Exception as err:
print(err)
5.运行结果
6.心得体会
- 在上一题复现代码的基础上，进一步熟悉了数据库相关进本操作;
- 了解了浏览器
F12抓包操作流程。
三、作业③
| 序号 |
学校 |
总分 |
| 1 |
清华大学 |
969.2 |
| 2 |
... |
... |
1.解析网页
- 调试分析gif
2.构造正则表达式
reg1 = 'univNameCn:"(.*?)"' #学校
reg2 = 'score:(.*?),' #总分
3.创建数据库
self.cursor.execute("create table university(sTop varchar(16),"
"sName varchar(64),sScore varchar(16),"
"constraint pk_university primary key (sName))")
4.获取并打印信息
- 注意到有些学校的score值为空，这里采用了
is_number()判断其是否为数值，若不是则替换为-
def is_number(self, s):
try:
float(s)
return True
except ValueError:
pass
try:
import unicodedata
unicodedata.numeric(s)
return True
except (TypeError, ValueError):
pass
return False
打印信息
reg1 = 'univNameCn:"(.*?)"'
reg2 = 'score:(.*?),'
name = re.findall(reg1, resp.text)
score = re.findall(reg2, resp.text)
# 中西文混排时，要使用中文字符空格填充chr(12288)
tplt = "{0:^2}\t{1:{3}^10}\t{2:{3}^3}"
# 对中文输出的列，进行用第6个参数即中文空格chr（12288）填充
print(tplt.format("排名", "学校名称", "总分", chr(12288)))
for i in range(len(name)):
if self.is_number(score[i]):
#判断是否为数值
score[i] = score[i]
else:
score[i] = '-'
self.db.insert(i + 1, name[i], score[i])
print(tplt.format(i + 1, name[i], score[i], chr(12288)))
5.运行结果
6.心得体会
- 本次获取大学排名与之前略有不同，需先进行浏览器调试分析，通过正则匹配获取全部排名。