import requests
from bs4 import BeautifulSoup

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
                  ' Chrome/52.0.2743.116 Safari/537.36 Edge/15.16193'}

start_url = 'https://www.pythonscraping.com'

r = requests.get(start_url, headers=headers)
soup = BeautifulSoup(r.text, 'lxml')
# 获取所有img标签
img_tags = soup.find_all('img')
for tag in img_tags:
    print(tag['src'])

import csv
import requests
from bs4 import BeautifulSoup

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
                  ' Chrome/52.0.2743.116 Safari/537.36 Edge/15.16193'}

url = 'https://en.wikipedia.org/wiki/Comparison_of_text_editors'
r = requests.get(url, headers=headers)
soup = BeautifulSoup(r.text, 'lxml')
# 只要第一个表格
rows = soup.find('table', class_='wikitable').find_all('tr')

# csv写入时候每写一行会有一空行被写入，所以设置newline为空
with open('editors.csv', 'w', newline='', encoding='utf-8') as f:
    writer = csv.writer(f)
    for row in rows:
        csv_row = []
        for cell in row.find_all(['th', 'td']):
            csv_row.append(cell.text)

        writer.writerow(csv_row)

from io import StringIO
import csv
import requests

csv_data = requests.get('http://pythonscraping.com/files/MontyPythonAlbums.csv')
data_file = StringIO(csv_data.text)
reader = csv.reader(data_file)

for row in reader:
    print(row)

from io import StringIO
import csv
import requests

csv_data = requests.get('http://pythonscraping.com/files/MontyPythonAlbums.csv')
data_file = StringIO(csv_data.text)
reader = csv.DictReader(data_file)
# 查看Key
print(reader.fieldnames)

for row in reader:
    print(row['Year'], row['Name'], sep=': ')

import pymysql

conn = pymysql.connect(host='localhost', user='root', password='admin', db='example',charset='utf8')

cur = conn.cursor()

try:
    # 上面填了参数这句就不是必须的
    # cur.execute('USE example')
    cur.execute('SELECT * FROM pages')
    print(cur.fetchone())
finally:
    cur.close()
    conn.close()

import re
import random

import pymysql
import requests
from bs4 import BeautifulSoup

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
                  ' Chrome/52.0.2743.116 Safari/537.36 Edge/15.16193'}

conn = pymysql.connect(host='localhost', user='root', password='admin', db='example', charset='utf8')
cur = conn.cursor()

# 存到数据库
def store(title, content):
    try:
        cur.execute(f"INSERT INTO pages(title, content) VALUES('{title}', '{content}');")
    except Exception as e:
        print(e)
    else:
        conn.commit()

# 获得页面内所有词条的链接
def get_links(article_url):
    r = requests.get('https://en.wikipedia.org' + article_url, headers=headers)
    soup = BeautifulSoup(r.text, 'lxml')
    title = soup.h1.string
    content = soup.find('div', id='mw-content-text').find('p').text
    store(title, content)
    links = soup.find('div', id='bodyContent').find_all('a', href=re.compile('^/wiki/[^:/]*$'))
    return links

link_list = get_links('/wiki/Kevin_Bacon')
try:
    while len(link_list) > 0:
        new_article = random.choice(link_list).get('href')
        print(new_article)
        link_list = get_links(new_article)
finally:
    cur.close()
    conn.close()

import re

import pymysql
import requests
from bs4 import BeautifulSoup

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
                  ' Chrome/52.0.2743.116 Safari/537.36 Edge/15.16193'}

conn = pymysql.connect(host='localhost', user='root', password='admin', db='wiki', charset='utf8')
cur = conn.cursor()

def insert_page_if_not_exists(url):
    cur.execute(f"SELECT * FROM pages WHERE url='{url}';")
    # 这条url没有插入的话
    if cur.rowcount == 0:
        # 那就插入
        cur.execute(f"INSERT INTO pages(url) VALUES('{url}');")
        conn.commit()
        # 刚插入数据的id
        return cur.lastrowid
    # 否则已经存在这条数据，因为url一般是唯一的，所以获取一个就行，取脚标0是获得id
    else:
        return cur.fetchone()[0]

def insert_link(from_page, to_page):
    print(from_page, ' -> ', to_page)
    cur.execute(f"SELECT * FROM links WHERE fromId={from_page} AND toId={to_page};")
    # 如果查询不到数据，则插入，插入需要两个pages的id，即两个url
    if cur.rowcount == 0:
        cur.execute(f"INSERT INTO links(fromId, toId) VALUES({from_page}, {to_page});")
        conn.commit()

# 链接去重
pages = set()

# 得到所有链接
def get_links(page_url, recursion_level):
    global pages
    if recursion_level == 0:
        return
    # 这是刚插入的链接
    page_id = insert_page_if_not_exists(page_url)
    r = requests.get('https://en.wikipedia.org' + page_url, headers=headers)
    soup = BeautifulSoup(r.text, 'lxml')
    link_tags = soup.find_all('a', href=re.compile('^/wiki/[^:/]*$'))
    for link_tag in link_tags:
        # page_id是刚插入的url，参数里再次调用了insert_page...方法，获得了刚插入的url里能去往的url列表
        # 由此形成联系，比如刚插入的id为1，id为1的url里能去往的id有2、3、4...，则形成1 -> 2, 1 -> 3这样的联系
        insert_link(page_id, insert_page_if_not_exists(link_tag['href']))

        if link_tag['href'] not in pages:
            new_page = link_tag['href']
            pages.add(new_page)
            # 递归查找, 只能递归recursion_level次
            get_links(new_page, recursion_level - 1)

if __name__ == '__main__':
    try:
        get_links('/wiki/Kevin_Bacon', 5)
    except Exception as e:
        print(e)
    finally:
        cur.close()
        conn.close()