起始url的调度原理

# -*- coding: utf-8 -*-
import scrapy
from xdb.items import XdbItem
from scrapy.dupefilters import RFPDupeFilter
from scrapy.http.cookies import CookieJar
from scrapy.http import Request
from urllib.parse import urlencode

class ChoutiSpider(scrapy.Spider):
    name = chouti
    allowed_domains = [chouti.com]
    start_urls = [http://chouti.com/]
    # start_urls = [‘http://127.0.0.1:80/app01/login/‘]

    ‘‘‘
    scrapy引擎来爬虫中取起始URL:
        1. 调用start_request并获取返回值
        2. v = iter(返回值)
        3. req1 = 执行 v.__next__()
           req2 = 执行 v.__next__()
           req3 = 执行 v.__next__()
    ‘‘‘
    def start_requests(self):
        # 方式一:
        for url in self.start_urls:
            yield Request(url=url, callback=self.parse)
        # 方式二:
        req_list = []
        for url in self.start_urls:
            req_list.append(Request(url=url))
        return req_list

 

起始url的调度原理

上一篇:.NetCore 之AOP扩展ExceptionFilter


下一篇:js判断是否是app,及版本号