深度好文：
from scrapy.contrib.spiders.init import InitSpider
from scrapy.http import Request, FormRequest
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.contrib.spiders import Rule class MySpider(InitSpider):
name = 'myspider'
allowed_domains = ['domain.com']
login_page = 'http://www.domain.com/login'
start_urls = ['http://www.domain.com/useful_page/',
'http://www.domain.com/another_useful_page/'] rules = (
Rule(SgmlLinkExtractor(allow=r'-\w+.html$'),
callback='parse_item', follow=True),
) def init_request(self):
"""This function is called before crawling starts."""
return Request(url=self.login_page, callback=self.login) def login(self, response):
"""Generate a login request."""
return FormRequest.from_response(response,
formdata={'name': 'herman', 'password': 'password'},
callback=self.check_login_response) def check_login_response(self, response):
"""Check the response returned by a login request to see if we are
successfully logged in.
"""
if "Hi Herman" in response.body:
self.log("Successfully logged in. Let's start crawling!")
# Now the crawling can begin..
self.initialized()
else:
self.log("Bad times :(")
# Something went wrong, we couldn't log in, so nothing happens. def parse_item(self, response): # Scrape data from page
备注: 该代码片段来自于: http://www.sharejs.com/codes/python/8544 使用header
request_headers = { 'User-Agent': 'PeekABoo/1.3.7' }
request = urllib2.Request('http://sebsauvage.net', None, request_headers)
urlfile = urllib2.urlopen(request)