# -*- coding: utf-8 -*-

 # scrapy爬取极客学院全部课程

 import scrapy

 from pyquery import PyQuery as pq

 from jike.items import JikeItem

 class JikespiderSpider(scrapy.Spider):

     name = "jikespider"

     allowed_domains = ["www.jikexueyuan.com"]

     base_url = 'http://www.jikexueyuan.com/course/?pageNum='

     def start_requests(self):

         for page_num in range(1,96):

             url = self.base_url + str(page_num)

             yield scrapy.Request(url, callback=self.parse_index)

     def parse_index(self, response):

         doc = pq(response.text)

         lis = doc('.lesson-list .cf li').items()

         # pyquery心得, 以为pyquery有点问题而导致无法遍历数据结构,

         # 研究发现是'http:' + item('.lessonimg-box a').attr('href')

         # 的问题, href是相对路径没有得到一个有效的请求链接

         for item in lis:

             detail_url = 'http:' + item('.lessonimg-box a').attr('href')

             yield scrapy.Request(url=detail_url,callback=self.parse_detail)

     def parse_detail(self, response):

         item = JikeItem()

         doc = pq(response.text)

         item['title'] = doc('.lesson-teacher .bc-box h2').text()

         item['time'] = doc('.lesson-teacher .bc-box .timebox').text()

         item['content'] = doc('.lesson-teacher .infor-content').text()

         yield item