import requests

import re

import pandas as pd

def retrieve_dji_list():

    try:

        r = requests.get('https://money.cnn.com/data/dow30/')

    except ConnectionError as err:

        print(err)

    search_pattern = re.compile('class="wsod_symbol">(.*?)<\/a>.*?<span.*?">(.*?)<\/span>.*?\n.*?class="wsod_stream">(.*?)<\/span>')

    dji_list_in_text = re.findall(search_pattern, r.text)

    dji_list = []

    for item in dji_list_in_text:

        dji_list.append([item[0], item[1], float(item[2])])

    return dji_list

dji_list = retrieve_dji_list()

djidf = pd.DataFrame(dji_list)

print(djidf)

cols=['code','name','lasttrade']

djidf.columns=cols # 改变列名

djidf.index=range(1,len(djidf)+1)

djidf.code   # 获取code列+index

djidf['code']  # 获取code列 , 两者同功能

djidf.loc[1:5,]  # 前5行

djidf.loc[:,['code','lasttrade']] #所有行

djidf.loc[1:5,['code','lasttrade']] #1-5行, loc表示标签index

djidf.loc[1,['code','lasttrade']] #1行

djidf.at[1,'lasttrade']  # 只有一个值的时候可以用at

djidf.iloc[2:4,[0,2]]  # 表示物理文职, 并且4取不到, 就只有第三行第四行

djidf.iat[1,2]  # 单个值

djidf.lasttrade.mean()  # 121.132

djidf[djidf.lasttrade>=180].name

tempdf=djidf.sort_values(by='lasttrade',ascending=False)

tempdf[:3].name

df=pd.DataFrame(np.random.randn(3,3),index=['c','b','a'],columns=list('xyz'))

df.sort_index()  # 根据index 进行排序

import requests

import re

import json

import pandas as pd

from datetime import  date

def retrieve_quotes_historical(stock_code):

    quotes = []

    url = 'https://finance.yahoo.com/quote/%s/history?p=%s' % (stock_code, stock_code)

    try:

        r = requests.get(url)

    except ConnectionError as err:

        print(err)

    m = re.findall('"HistoricalPriceStore":{"prices":(.*?),"isPending"', r.text)

    if m:

        quotes = json.loads(m[0])       # m = ['[{...},{...},...]']

        quotes = quotes[::-1]         # 原先数据为date最新的在最前面

    return  [item for item in quotes if not 'type' in item]

quotes = retrieve_quotes_historical('AXP')

list1=[]

for i in range(len(quotes)):

    x=date.fromtimestamp(quotes[i]['date'])

    y=date.strftime(x,'%Y-%m-%d')

    list1.append(y)

quotesdf_ori=pd.DataFrame(quotes,index=list1)

quotesdf_m = quotesdf_ori.drop(['adjclose'], axis = 1)  #删除adjclose列

quotesdf=quotesdf_m.drop(['date'],axis=1)

print(quotesdf)

quotesdf[(quotesdf.index>='2017-03-01') & (quotesdf.index<='2017-03-31')]

quotesdf[(quotesdf.index>='2017-11-30') & (quotesdf.index<='2018-03-31')&

(quotesdf.close>=90)]

len(quotesdf.close>quotesdf.open) 

import numpy as np

status=np.sign(np.diff(quotesdf.close))

status  # 250 的长度, 比quotesdf 少1

status[np.where(status==1)].size  # np.where(status==1)是由下标构成的array

# 

import pandas as pd

import numpy as np

index_df = pd.DataFrame(np.random.rand(3,3), index=['a','b','c'], columns=['index_1','index_2','index_3'])

index_df.describe() # 超级强大

t=quotesdf[(quotesdf.index>='2018-01-01') & (quotesdf.index<'2018-02-01')]

len(t)  #21

import time

list2=[]

for i in range(len(quotesdf)):

    temp=time.strptime(quotesdf.index[i],'%Y-%m-%d')

    list2.append(temp.tm_mon)  # 取月份

tempdf=quotesdf.copy()

tempdf['month']=list2  # 新增一列月份的数据

print(tempdf['month'].value_counts())  # 计算每个月的出现次数

# 统计每一月的股票开盘天数

x=tempdf.groupby('month').count()

# 统计近一年每个月的成交量

tempdf.groupby('month').sum().volume

# 先每个月进行求和, 但是这些对其他列也进行了求和, 属于无效计算, 如何避免?

tempdf.groupby('month').volume.sum()  # 交换顺序即可

def f(df):

     return df.age.count()

data_df.groupby('taste of mooncake').apply(f)   

# append

p=quotesdf[:2]

q=quotesdf['2018-01-01':'2018-01-05']

p.append(q)

# concat

pieces=[tempdf[:5],tempdf[len(tempdf)-5:]]

pd.concat(pieces)

piece1=quotesdf[0:3]

piece2=tempdf[:3]

pd.concat([piece1,piece2],ignore_index=True)

#join 两个dataframe要有共同的字段(列名)

#djidf: code/name

#AKdf:  volume/code/month

# 合并之后的字段: code/name/volume/month

pd.merge(djidf.drop(['lasttrade'],axis=1),AKdf, on='code')