#创建爬虫
scrapy genspider randomIp_spider "taobao.com"
#把需要请求的url放到一个混淆的url请求list中去，避免被监测到总是访问此页面
import random url_list = [
'https://detail.tmall.com/item.htm?id=522194707780&ali_refid=a3_430583_1006:1109696291:N:%E6%B6%88%E9%98%B2%E5%BA%94%E6%80%A5%E7%81%AF:eb9682757281a9ec406cb4647d3f584a&ali_trackid=1_eb9682757281a9ec406cb4647d3f584a&spm=a230r.1.14.3',
'https://item.taobao.com/item.htm?spm=a219r.lmn002.14.1.f3b87156TcpPbp&id=587398066660&ns=1&abbucket=16',
'https://item.taobao.com/item.htm?spm=a230r.1.14.50.1af3248cr0GGyM&id=576997844987&ns=1&abbucket=16#detail'
]
#随机去一个访问链接
start_urls = random.choice(url_list)
#到middlewares.py文件中去
#设置随机请求头
class UserAgentDownloadMiddleware(object):
USER_AGENTS = [
'Mozilla/5.0 (Macintosh; U; PPC Mac OS X; pl-PL; rv:1.0.1) Gecko/20021111 Chimera/0.6',
'Mozilla/5.0 (Macintosh; U; PPC Mac OS X; pl-PL; rv:1.0.1) Gecko/20021111 Chimera/0.6',
'Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US; rv:1.0.1) Gecko/20021111 Chimera/0.6',
'Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US; rv:1.0.1) Gecko/20021104 Chimera/0.6',
'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20030111 Chimera/0.6',
'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20030109 Chimera/0.6',
'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20021220 Chimera/0.6',
'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20021216 Chimera/0.6'
] def process_request(self, request, spider):
user_agent = random.choice(self.USER_AGENTS)
request.headers['User-Agent'] = user_agent
#设置随机ip
class IPProxyDownloadMiddleware(object):
PROXIES = [
'222.190.163.141:45334',
'183.143.73.146:31998',
'115.216.58.182:43060',
'116.209.129.167:27158',
'60.167.23.29.205:44728'
] def process_request(self, request, spider):
proxy = random.choice(self.PROXIES)
print('+' * )
print(proxy)
request.meta['proxy'] = proxy f = open("texr.json", encoding='utf-8')
setting = json.load(f)
family = setting['BaseSettings']['size']
#到settings.py文件中去,修改如下配置
DOWNLOADER_MIDDLEWARES = {
#随机请求头
'taobao_for_attack.middlewares.UserAgentDownloadMiddleware': ,、
#随机ip
'taobao_for_attack.middlewares.IPProxyDownloadMiddleware': ,
}