#!/usr/bin/env python

 # -*- coding: utf-8 -*-

 import sys

 reload(sys)

 sys.setdefaultencoding('utf-8')  #解决编码问题

 """一个简单的从豆瓣网获取电影标签的示例,

 1. 首先获取所有标签以及它所对应的请求连接,

    获取所有标签的请求链接是https://movie.douban.com/tag/?view=cloud

 2. 请求每个标签对应的链接获取该标签下所有电影信息

 """

 import urllib

 import urllib2

 import re

 import codecs

 # BeautifulSoup 用于解析html使用，本程序中使用lxml作为解析器

 from bs4 import BeautifulSoup

 # url正则表达式，用于将url分割为多个部分，对其中某些部分进行url编码使用

 URL_RGX = re.compile(r'(?P<prefix>https?://)(?P<mid>[^?]+)(?P<has_key>\??)(?P<parms>[^?]*)')

 DOUBAN_TAG_URL = r'https://movie.douban.com/tag/?view=cloud'

 # url请求

 def get_request(url):

     # 对url特定部分进行编码

     matches = URL_RGX.match(url)

     if matches:

         dic = matches.groupdict()

         prefix = dic.get('prefix', '')

         mid = urllib.quote(dic.get('mid', ''))

         has_key = dic.get('has_key', '')

         parms = dic.get('parms', '')

         if parms != '':

             tmp_dic = dict()

             for kv in parms.split('&'):

                 k, v = kv, ''

                 if '=' in kv:

                     k, v = kv.split('=', 1)

                 tmp_dic[k] = v

             parms = urllib.urlencode(tmp_dic)

         url = '{0}{1}{2}{3}'.format(prefix, mid, has_key, parms)

     else:

         url = None

     if url:

         try:

             res = urllib.urlopen(url)

             if res and res.getcode() == 200:

                 return res.read()

         except Exception as e:

             pass

     return ''

 # 获取所有标签信息，返回值为列表，列表中每个元素格式为(标签名, 标签url地址)

 def get_all_tags():

     all_tags = list()

     page_text = get_request(DOUBAN_TAG_URL)

     soup = BeautifulSoup(page_text, 'lxml')

     tags = soup.find_all('a', attrs={'class': 'tag'})

     for tag in tags:

         tag_name = tag.string

         # 将原有链接中的?focus=替换为空，替换后url为分页显示地址

         tag_url = tag['href'].replace('?focus=', '')

         all_tags.append((tag_name, tag_url))

     return all_tags

 # 解析单个电影节点信息，返回电影信息字典

 def parse_movie_tag(movie_tag):

     # 下面15行代码好烂，下次改下它

     movie_title_tag = movie_tag.find('a', attrs={'class': 'title'})

     movie_title = '无名电影'

     movie_detail_url = '无链接'

     if movie_title_tag:

         movie_title = movie_title_tag.string.strip()

         movie_detail_url = movie_title_tag['href'].strip()

     # movie_detail_url = movie_tag.find('a', attrs={'class': 'title'})['href']

     movie_desc_tag = movie_tag.find('div', attrs={'class': 'desc'})

     movie_desc = '无描述信息'

     if movie_desc_tag:

         movie_desc = movie_desc_tag.string.strip()

     movie_rating_nums_tag = movie_tag.find('span', attrs={'class': 'rating_nums'})

     movie_rating_nums = '无评分信息'

     if movie_rating_nums_tag:

         movie_rating_nums = movie_rating_nums_tag.string.strip()

     return {'title': movie_title, 'detail_url': movie_detail_url,

             'desc': movie_desc, 'rating_nums': movie_rating_nums}

 # 获取当前页面中的电影信息节点

 def get_current_page_movies(page_url):

     page_text = get_request(page_url)

     soup = BeautifulSoup(page_text, 'lxml')

     movies = soup.find_all('dd')

     movie_info_lst = list()

     for movie in movies:

         # 将解析后的电影信息加入列表中

         movie_info_lst.append(parse_movie_tag(movie))

     return movie_info_lst

 if __name__ == '__main__':

     # 获取所有标签信息

     all_tags = get_all_tags()

     # 标签信息存到文件中

     all_tags_file = codecs.open('all_tags_info.txt', 'wb')

     all_tags_file.write('标签名\t标签url地址\r\n')

     for tag_name, tag_url in all_tags:

         all_tags_file.write('{0}\t{1}\r\n'.format(tag_name, tag_url))

         all_tags_file.flush()

     all_tags_file.close()

     # 获取每个标签下的所有电影

     for tag_name, tag_url in all_tags:

         movie_infos_file = codecs.open(tag_name + '.txt', 'wb')

         movie_infos_file.write('电影名\t标签名\t电影描述\t评分\t详细链接\r\n')

         start = 0

         while True:

             target_url = '{base_url}?start={start}'.format(base_url=tag_url,

                                                            start=start)

             movies = get_current_page_movies(target_url)

             for movie in movies:

                 title = movie.get('title', '')

                 detail_url = movie.get('detail_url', '')

                 desc = movie.get('desc', '')

                 rating_nums = movie.get('rating_nums', '')

                 movie_infos_file.write('{0}\t{1}\t{2}\t{3}\t{4}\r\n'.format(title,

                                                                           tag_name,

                                                                           desc,

                                                                           rating_nums,

                                                                           detail_url))

                 movie_infos_file.flush()

             # 计算当前页电影总个数，因单页只显示15条信息，

             # 所以如果单页电影数小于15则表示无后续页面，则跳出循环不再请求此分类

             current_page_movies_count = len(movies)

             if current_page_movies_count < 15:

                 break

             # 请求下一页标记数

             start += current_page_movies_count

         movie_infos_file.close()