spider csdn博客和quantstart文章

功能

1. 提取csdn博客文章
2. 提取quantstart.com 博客文章, Micheal Hall-Moore 创办的网站

特色功能就是: 想把原来文章里的格式/样式(段落, 标题等等排版信息)保留到word文档里. 不仅仅是把文本提取出来.

目前能够全部得到文章信息. 而且博文里的段落和小标题信息也都保留了下来

TODO:

把它们写入word文档, 同时也要保留段落以及样式信息.

(转码到docx的部分留到以后实现)

遇到的问题以及解决办法

csdn博文用常规的requests库, 得到的是乱码.

多次尝试了指定编码, 也没有得到正解.

无奈只好改用selenium这个大笨蛋获取.

如果读者朋友知道requests的实现方案, 请不吝赐教.

关于用webdriver定位页面元素的技巧:

用wait...untill 组合手段来解决. 完美解决无法定位问题/或者简单设定超时等等时间.

WebDriverWait(brower, timeOut).until(lambda x: x.title)

WebDriverWait(brower, timeOut).until(lambda x: x.page_source)

WebDriverWait(brower, timeOut).until(lambda x: x.find_element_by_class('abc'))

截图

代码

v 0.2

import os; type(os)

import time; type(time)

import re

anys = '.*?' # 任意长的字符串, 贪婪型的

import random; type(random)

import requests

from lxml import etree

from selenium import webdriver

chrome_options = webdriver.ChromeOptions()

chrome_options.binary_location = r'C:\Users\Administrator\AppData\Roaming\360se6\Application\360se.exe'

chrome_options.add_argument(r'--lang=zh-CN') # 这里添加一些启动的参数

import logging

logging.basicConfig(level=logging.INFO,

    format= '%(asctime)s - %(name)s - %(levelname)s : %(message)s',

    #format='%(asctime)s %(filename)s [line:%(lineno)d] %(levelname)s %(message)s',

    )

logger = logging.getLogger(__name__)

#logger.info("Start print log")

#logger.debug("Do something")

#logger.warning("Something maybe fail.")

#logger.info("Finish")

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.common.by import By

from selenium.webdriver.support import expected_conditions as EC

type(By)

type(EC)

#%%

def get_it_wait_untill(browser, element_func='title', sleep_time=80, arg=''):

    '''

    selenium内核的锁定页面元素, 然后取之. 比如:

        获取网页标题

        获取整个网页的源文件

        获取指定页面元素:

            by_id

            by_xpath

    Example:

        >>> get_it_wait_untill(browser, 'title')

        >>> get_it_wait_untill(browser, 'page_source')

        >>> get_it_wait_untill(browser, element_func='find_element_by_id',

                        arg='content_views',

                        )

        >>> get_it_wait_untill(browser, element_func='find_element_by_xpath',

                        arg='//section[@class="content article-content"]',

                        )

    '''

    prop = str(type(getattr(browser, element_func)))

    if prop == "<class 'str'>":

        element = WebDriverWait(browser, sleep_time).until(

                lambda x: getattr(x, element_func)

                )

    #elif callable(getattr(browser, element_func)):

    elif prop == "<class 'method'>":

        element = WebDriverWait(browser, sleep_time).until(

                lambda x: getattr(x, element_func)(arg)

                )

    return element

#%%

def get_quantstart_article(

        url = 'https://www.quantstart.com/articles/Beginners-Guide-to-Quantitative-Trading'

        ,

        sleep_time=40

        ,

        kernel='requests', # or 'selenium'

        xpath = '//section[@class="content article-content"]'

        ,

        ):

    '''quantstart blog by:  Micheal Hall-Moore

    '''

    #Beginner's Guide to Quantitative Trading | QuantStart

    logger.info(f'当前的url: {url}')

    if kernel=='requests':

        headers = {

                'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)',

        #        'Cookie'    : 'acw_tc=2760820715764664133076924e81d61fc0916132e54d88a183dbc0bd00aa01; acw_sc__v2=5df6f7edcaabd3ef5262a7a3d573f5675b385aec; uuid_tt_dd=10_17296990010-1576466413788-862921; dc_session_id=10_1576466413788.893537; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1576466384; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=6525*1*10_17296990010-1576466413788-862921; announcement=%257B%2522isLogin%2522%253Afalse%252C%2522announcementUrl%2522%253A%2522https%253A%252F%252Fblogdev.blog.csdn.net%252Farticle%252Fdetails%252F103053996%2522%252C%2522announcementCount%2522%253A0%252C%2522announcementExpire%2522%253A3600000%257D; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1576466781; c-login-auto=2; dc_tos=q2l5ox',

                }

        try:

            resp = requests.get(url, params=headers)

            if resp.status_code==200:

                logger.info(f'get网络请求的响应ok!!!, 响应的编码是{resp.encoding}')

                html = resp.text

                #html2 = str(resp.content, encoding='utf-8')

                #is_eqs = html==html2; print(is_eqs)

                tree = etree.HTML(html)

                article_tree = tree.xpath(xpath)[0]

                article_html = etree.tostring(article_tree,); type(article_html)

                article_unicode = etree.tounicode(article_tree)

                article_unicode = re.findall('<section.*?(<p>.*?)</section>',

                               article_unicode, re.S)[0]

                logger.info(article_unicode[-100:])

                return article_unicode.strip()

            else:

                logger.info(f'responce status code: {resp.status_code}')

                logger.error('requests.get()网络连接/网络请求不正常!!!')

                return False

        except Exception as ee:

            logger.error(f'Exception: {ee}')

            return False

    elif kernel=='selenium':

        browser = webdriver.Chrome(options=chrome_options)

        #timeout_wait = WebDriverWait(browser, 2*5) # 10sec

        browser.get(url)

        timeout_wait = WebDriverWait(browser, 2*5) # 10sec;

        type(timeout_wait)

        #title = browser.title

        title = get_it_wait_untill(browser, 'title')

        logger.info(f'网页的标题是: {title} ')

        html = get_it_wait_untill(browser, 'page_source') # 需要花点时间

        article_webelement = browser.find_element_by_xpath(xpath)

        article_text = article_webelement.text; type(article_text)

        logger.info('网页源码的长度和博文的长度分别是: {1} {0}'.

              format(len(article_text), len(html))

              )

        #    pattern = '<title>(.*?)</title>'

        #    pattern = '<p>(.*?)</p>'

        pattern = '</style></form>' + \

            '(.*?)' + \

            '<div class="col-md-4 book-card order-md-1">'

        a = re.findall(pattern, html, re.S)

        a = a[0]

        a = re.findall(f'{anys}(<p>{anys})</section>{anys}', a, re.S)[0]

        return a

    else:

        logger.error('出错了!!! 你必须指定正确的内核. 或者selenium 或者requests!')

        return False

#%%

def get_csdn_blog(

        url='https://blog.csdn.net/Lili_0820/article/details/70155949'

        ,

        sleep_time=40

        ,

        ):

    '''

    爬取csdn blog文章

    参数:

    url: str,

    sleep_time: int, wait time in seconds

    '''

    logger.info(f'当前的url: {url}')

    browser = webdriver.Chrome(options=chrome_options)

    #timeout_wait = WebDriverWait(browser, 2*5) # 10sec

    browser.get(url)

    timeout_wait = WebDriverWait(browser, sleep_time) # 10sec;

    type(timeout_wait)

    '''

    我们需要确保: 网页信息已经全部加载, 否则可能提取不到有用信息.

    Sets a sticky timeout to implicitly wait for an element to be found,

    or a command to complete.

    This method only needs to be called one time per session.

    当浏览器(webdriver实例)在定位元素的时候,

    我们可以设置一个隐式的超时等待时间,

    如果超过这个设定的时间还不能锁定元素, 那么就报错或者继续执行.

    本方法在整个对话期内, 只需调用一次.

    '''

    browser.implicitly_wait(200)

    '''

    i=0

    while 1: # 有可能是一个无限循环, 改用: wait...untill

        title = browser.title

        # selenium 取数据时的等待时间间隔是: 0.5sec

        # poll_frequency=0.5

        i+=1

        logger.info(f'提取网页标题: 定位title元素的次数: \n{i} {title}')

        if len(title)>=5 :

            break

    '''

    title = WebDriverWait(browser, sleep_time).until(lambda x: x.title)

    logger.info(f'提取网页标题: {title}')

    html= WebDriverWait(browser, sleep_time).until(lambda x: x.page_source)

    #html = browser.page_source

    #需要花点时间

    #time.sleep(sleep_time) # 太粗暴简单了

    content = browser.find_element_by_id('content_views') # selenium.webelement

    text = content.text; type(text)

    logger.info('网页源码的长度和博文的长度分别是: {1} {0}'.

          format(len(text), len(html))

          )

    pattern = 'id="content_views" class="markdown_views.*?>' + \

        '(.*?)' + \

        '<link href="https://csdnimg.cn/release/' + \

        'phoenix/mdeditor/markdown_views'

    a = re.findall(pattern, html, re.S)

    a = a[0]

    a = re.findall(f'{anys}(<p>{anys})</div>{anys}', a, re.S)[0]

    '''

    tree = etree.HTML(html)

    tree_cv = tree.xpath('//div[@id="content_views"]')[0]

    txt = tree_cv.xpath('*/text()')

    txt2 = tree_cv.xpath('*//text()')

    txt22 = '\n'.join(txt2)

    html_cv = etree.tostring(tree_cv)

    '''

    browser.close()

    return a

if __name__=='__main__':

    pass

#    blog2r=get_quantstart_article()

#    blog2r=get_quantstart_article(kernel='requests')

#    blog2r=get_quantstart_article(url='http://www.ba', kernel='requests')

#    blog2s=get_quantstart_article(kernel='selenium')

#    blog2=get_quantstart_article(kernel='req')

#    blog1 = get_csdn_blog(sleep_time=80)

v 0.1

import os; type(os)

import time

import re

anys = '.*?' # 任意长的字符串, 贪婪型的

import random; type(random)

import requests

from lxml import etree

from selenium import webdriver

chrome_options = webdriver.ChromeOptions()

chrome_options.binary_location = r'C:\Users\Administrator\AppData\Roaming\360se6\Application\360se.exe'

chrome_options.add_argument(r'--lang=zh-CN') # 这里添加一些启动的参数

import logging

logging.basicConfig(level=logging.INFO,

    format= '%(asctime)s - %(name)s - %(levelname)s : %(message)s',

    #format='%(asctime)s %(filename)s [line:%(lineno)d] %(levelname)s %(message)s',

    )

logger = logging.getLogger(__name__)

#logger.info("Start print log")

#logger.debug("Do something")

#logger.warning("Something maybe fail.")

#logger.info("Finish")

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.common.by import By

from selenium.webdriver.support import expected_conditions as EC

type(By)

type(EC)

def get_quantstart_article(

        url = 'https://www.quantstart.com/articles/Beginners-Guide-to-Quantitative-Trading'

        ,

        sleep_time=40

        ,

        kernel='requests', # or 'selenium'

        xpath = '//section[@class="content article-content"]'

        ,

        ):

    '''quantstart blog by:  Micheal Hall-Moore

    '''

    #Beginner's Guide to Quantitative Trading | QuantStart

    logger.info(f'当前的url: {url}')

    if kernel=='requests':

        headers = {

                'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)',

        #        'Cookie'    : 'acw_tc=2760820715764664133076924e81d61fc0916132e54d88a183dbc0bd00aa01; acw_sc__v2=5df6f7edcaabd3ef5262a7a3d573f5675b385aec; uuid_tt_dd=10_17296990010-1576466413788-862921; dc_session_id=10_1576466413788.893537; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1576466384; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=6525*1*10_17296990010-1576466413788-862921; announcement=%257B%2522isLogin%2522%253Afalse%252C%2522announcementUrl%2522%253A%2522https%253A%252F%252Fblogdev.blog.csdn.net%252Farticle%252Fdetails%252F103053996%2522%252C%2522announcementCount%2522%253A0%252C%2522announcementExpire%2522%253A3600000%257D; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1576466781; c-login-auto=2; dc_tos=q2l5ox',

                }

        try:

            resp = requests.get(url, params=headers)

            if resp.status_code==200:

                logger.info(f'get网络请求的响应ok!!!, 响应的编码是{resp.encoding}')

                html = resp.text

                #html2 = str(resp.content, encoding='utf-8')

                #is_eqs = html==html2; print(is_eqs)

                tree = etree.HTML(html)

                article_tree = tree.xpath(xpath)[0]

                article_html = etree.tostring(article_tree,); type(article_html)

                article_unicode = etree.tounicode(article_tree)

                article_unicode = re.findall('<section.*?(<p>.*?)</section>',

                               article_unicode, re.S)[0]

                logger.info(article_unicode[-100:])

                return article_unicode.strip()

            else:

                logger.info(f'responce status code: {resp.status_code}')

                logger.error('requests.get()网络连接/网络请求不正常!!!')

                return False

        except Exception as ee:

            logger.error(f'Exception: {ee}')

            return False

    elif kernel=='selenium':

        browser = webdriver.Chrome(options=chrome_options)

        #timeout_wait = WebDriverWait(browser, 2*5) # 10sec

        browser.get(url)

        timeout_wait = WebDriverWait(browser, 2*5) # 10sec;

        type(timeout_wait)

        title = browser.title

        print('网页的标题是: ', title)

        html = browser.page_source # 需要花点时间

        time.sleep(sleep_time)

        article_webelement = browser.find_element_by_xpath(xpath)

        article_text = article_webelement.text; type(article_text)

        print('网页源码的长度和博文的长度分别是: {1} {0}'.

              format(len(article_text), len(html))

              )

        #    pattern = '<title>(.*?)</title>'

        #    pattern = '<p>(.*?)</p>'

        pattern = '</style></form>' + \

            '(.*?)' + \

            '<div class="col-md-4 book-card order-md-1">'

        a = re.findall(pattern, html, re.S)

        a = a[0]

        a = re.findall(f'{anys}(<p>{anys})</section>{anys}', a, re.S)[0]

        return a

    else:

        logger.error('出错了!!! 你必须指定正确的内核. 或者selenium 或者requests!')

        return False

#%%

def get_csdn_blog(

        url='https://blog.csdn.net/Lili_0820/article/details/70155949'

        ,

        sleep_time=40

        ,

        ):

    '''

    爬取csdn blog文章

    参数:

    url: str,

    sleep_time: int, wait time in seconds

    '''

    logger.info(f'当前的url: {url}')

    browser = webdriver.Chrome(options=chrome_options)

    #timeout_wait = WebDriverWait(browser, 2*5) # 10sec

    browser.get(url)

    timeout_wait = WebDriverWait(browser, sleep_time) # 10sec;

    type(timeout_wait)

    '''

    我们需要确保: 网页信息已经全部加载, 否则可能提取不到有用信息.

    Sets a sticky timeout to implicitly wait for an element to be found,

    or a command to complete.

    This method only needs to be called one time per session.

    当浏览器(webdriver实例)在定位元素的时候,

    我们可以设置一个隐式的超时等待时间,

    如果超过这个设定的时间还不能锁定元素, 那么就报错或者继续执行.

    本方法在整个对话期内, 只需调用一次.

    '''

    browser.implicitly_wait(200)

    '''

    i=0

    while 1: # 有可能是一个无限循环, 改用: wait...untill

        title = browser.title

        # selenium 取数据时的等待时间间隔是: 0.5sec

        # poll_frequency=0.5

        i+=1

        logger.info(f'提取网页标题: 定位title元素的次数: \n{i} {title}')

        if len(title)>=5 :

            break

    '''

    title = WebDriverWait(browser, sleep_time).until(lambda x: x.title)

    logger.info(f'提取网页标题: {title}')

    html= WebDriverWait(browser, sleep_time).until(lambda x: x.page_source)

    #html = browser.page_source

    #需要花点时间

    #time.sleep(sleep_time) # 太粗暴简单了

    content = browser.find_element_by_id('content_views') # selenium.webelement

    text = content.text; type(text)

    logger.info('网页源码的长度和博文的长度分别是: {1} {0}'.

          format(len(text), len(html))

          )

    pattern = 'id="content_views" class="markdown_views.*?>' + \

        '(.*?)' + \

        '<link href="https://csdnimg.cn/release/' + \

        'phoenix/mdeditor/markdown_views'

    a = re.findall(pattern, html, re.S)

    a = a[0]

    a = re.findall(f'{anys}(<p>{anys})</div>{anys}', a, re.S)[0]

    '''

    tree = etree.HTML(html)

    tree_cv = tree.xpath('//div[@id="content_views"]')[0]

    txt = tree_cv.xpath('*/text()')

    txt2 = tree_cv.xpath('*//text()')

    txt22 = '\n'.join(txt2)

    html_cv = etree.tostring(tree_cv)

    '''

    browser.close()

    return a

if __name__=='__main__':

#    blog2r=get_quantstart_article()

#    blog2r=get_quantstart_article(kernel='requests')

#    blog2r=get_quantstart_article(url='http://www.ba', kernel='requests')

#    blog2s=get_quantstart_article(kernel='selenium')

#    blog2=get_quantstart_article(kernel='req')

    blog1 = get_csdn_blog(sleep_time=80)