1、摘要
本文主要讲解：python_经验模态分解EMD_长短期记忆模型LSTM_公交短时客流预测
主要思路：
- 整理特征：天气、风力、时间、工作日和非工作日、节假日和非节假日、温度等
- 对客流量进行经验模态分解EMD
- 构建LSTM网络，优化器选择Adam
- reshape训练集和测试集，适配LSTM网络的输入尺寸
- 设置 batch_size和epochs，开始训练
- 评估模型、保存模型、画出模型预测结果的图
2、数据介绍
3、完整代码
import math
import os
import numpy as np
import pandas as pd
from PyEMD import EMD
from keras import Sequential
from keras.layers import Dense, LSTM
from keras.layers import Dropout
from sklearn import preprocessing
# 定义多通道特征组合模型
from sklearn.metrics import mean_squared_error
def build_model():
d = 0.2
neurons = [128, 128, 32, 1]
model_lstm = Sequential()
# 对每天61条记录进行分块
model_lstm.add(LSTM(neurons[0], input_shape=(61, 7), return_sequences=True))
model_lstm.add(Dropout(d))
model_lstm.add(LSTM(neurons[1], input_shape=(61, 1), return_sequences=False))
model_lstm.add(Dropout(d))
model_lstm.add(Dense(neurons[2], kernel_initializer="uniform", activation='relu'))
model_lstm.add(Dense(neurons[3], kernel_initializer="uniform", activation='linear'))
# adam = keras.optimizers.Adam(decay=0.2)
model_lstm.compile(loss='mse', optimizer='adam', metrics=['accuracy'])
model_lstm.summary()
return model_lstm
class DataLoader():
"""一个用于EMD-lstm模型数据加载和转换的类"""
def __init__(self, filename, cols, input_timesteps, seq_len):
"""
:param filename: the name of the file contains the data, type: .csv
:param split1: split the data into 2 parts: training, (validation, test)
|-------------------------------------------|-------------|--------------|
split1(0.7) split2(0.85)
:param cols: the features
:param input_timesteps: the length of looking back (1 month or 1 year), unit: hours
:param seq_len: the sum of input_timesteps and pre_len
"""
self.dataframe = pd.read_excel(filename)
self.test_data = pd.read_excel(r'上车测试数据.xlsx', usecols=range(1, 9))
self.test_data = pd.concat([self.test_data[:61], self.test_data], axis=0)
self.cols = cols
self.len_train_windows = None
self.input_timesteps = input_timesteps
self.seq_len = seq_len
print('the input cols are:', self.cols)
self.Normalization()
def scale_EMD(self):
import matplotlib.pyplot as plt
train_pro = self.cols[1:]
emd_array = self.dataframe['card_id'].values
self.IMFs = EMD().emd(emd_array)
# plt.plot(self.IMFs.reshape(12 * 5124, 1), color="red", label="Fitting Line", linewidth=2)
# plt.legend()
# plt.show()
# print('the signal is decomposed into ' + str(self.IMFs.shape[0]) + ' parts')
self.df_names_IMF = locals()
for ind, IMF in enumerate(self.IMFs):
IMF_name = 'IMF' + str(ind) + '_card_id'
data = {IMF_name: self.IMFs[ind]}
IMF_i = pd.DataFrame(data=data).reset_index()
fe = self.dataframe[train_pro].reset_index()
self.df_names_IMF['IMF' + str(ind)] = pd.merge(IMF_i, fe, on='index')
emd_test = self.test_data['card_id'].values
self.test_IMFs = EMD().emd(emd_test)
# plt.plot(self.test_IMFs.reshape(7 * 487, 1), color="orange", label="Fitting Line", linewidth=2)
# plt.legend()
# plt.show()
# print(self.test_IMFs)
# print('the signal is decomposed into ' + str(self.IMFs.shape[0]) + ' parts')
self.test_names_IMF = locals()
for ind, IMF in enumerate(self.test_IMFs):
IMF_name = 'IMF' + str(ind) + '_card_id'
data = {IMF_name: self.test_IMFs[ind]}
IMF_i = pd.DataFrame(data=data).reset_index()
fe = self.test_data[train_pro].reset_index()
self.test_names_IMF['IMF' + str(ind)] = pd.merge(IMF_i, fe, on='index')
def make_train_test_data(self):
ts = 61
train_x = self.data_train[:, 2:]
train_y = self.data_train[:, 1:2]
test_x = self.test_data[:, 2:]
test_y = self.test_data[:, 1:2]
# ############# 构建训练和预测集 ###################
ts_train_x = np.array([])
ts_train_y = np.array([])
ts_test_x = np.array([])
ts_test_y = np.array([])
# 构建训练数据集
print('训练数据的原始shape：', train_x.shape)
for i in range(train_x.shape[0]):
if i + ts == train_x.shape[0]:
break
ts_train_x = np.append(ts_train_x, train_x[i: i + ts, :])
ts_train_y = np.append(ts_train_y, train_y[i + ts])
# 构建预测数据集
print('预测数据的原始shape：', test_x.shape)
for i in range(test_x.shape[0]):
if i + ts == test_x.shape[0]:
break
ts_test_x = np.append(ts_test_x, test_x[i: i + ts, :])
ts_test_y = np.append(ts_test_y, test_y[i + ts])
return ts_train_x.reshape((train_x.shape[0] - ts, ts, train_x.shape[1])), ts_train_y, \
ts_test_x.reshape((test_x.shape[0] - ts, ts, test_x.shape[1])), ts_test_y
def Normalization(self):
'''
对训练数据进行规范化处理，并对验证和测试数据应用相同的量表
'''
self.scale_EMD()
IMF_number = 8
print('processing the data of IM' + str(IMF_number))
if IMF_number in range(self.IMFs.shape[0]):
self.data_train_original = self.df_names_IMF['IMF' + str(IMF_number)]
else:
print("Oops!IMF_number was no valid number. it must between 0 and " + str(self.IMFs.shape[0] - 1))
self.data_train_original = pd.concat([self.data_train_original[:61], self.data_train_original], axis=0)
self.min_max_scaler = preprocessing.MinMaxScaler()
self.data_train = self.min_max_scaler.fit_transform(self.data_train_original.values)
self.len_train = len(self.data_train_original)
IMF_test_number = 6
if IMF_test_number in range(self.test_IMFs.shape[0]):
self.test_original = self.test_names_IMF['IMF' + str(IMF_test_number)]
else:
print("Oops!IMF_number was no valid number. it must between 0 and " + str(self.test_IMFs.shape[0] - 1))
self.min_max_scaler = preprocessing.MinMaxScaler()
self.test_data = self.min_max_scaler.fit_transform(self.test_original.values)
self.len_test = len(self.test_original)
def model_score(model, X_train, y_train, X_test, y_test):
trainScore = model.evaluate(X_train, y_train, verbose=0)
print('Train Score: %.5f MSE (%.2f RMSE)' % (trainScore[0], math.sqrt(trainScore[0])))
testScore = model.evaluate(X_test, y_test, verbose=0)
print('Test Score: %.5f MSE (%.2f RMSE)' % (testScore[0], math.sqrt(testScore[0])))
def model_test_score(model, X_test, y_test):
y_hat = model.predict(X_test)
y_t = y_test.reshape(-1, 1)
temp = pd.DataFrame(y_hat)
temp['yhat'] = y_hat
temp['y'] = y_t
temp_rmse = np.sqrt(mean_squared_error(temp.y, temp.yhat))
temp_mse = mean_squared_error(temp.y, temp.yhat)
print('test RMSE: %.3f' % temp_rmse)
print('test MSE: %.3f' % temp_mse)
return temp_rmse, temp_mse
if __name__ == '__main__':
os.chdir(r'E:\项目文件\基于改进的LSTM短时客流预测\数据')
save_dir = 'E:\项目文件\基于改进的LSTM短时客流预测\emd-lstm模型\\'
# 每个时间序列块的数据行数
seq_len = 61
data = DataLoader(
filename=os.path.join(r'E:\项目文件\基于改进的LSTM短时客流预测\数据\\', r'上车训练数据.xlsx'),
cols=['card_id', 'feng_1.0', 'feng_2.0', 'feng_3.0', 'work_1', 'work_2', 'tianqi_average_scale',
'temperature_average_scale'],
input_timesteps=seq_len,
seq_len=61
)
train_x, train_y, val_x, val_y = data.make_train_test_data()
print('Starting training Model')
model = build_model()
model.fit(train_x, train_y, epochs=512, batch_size=32)
save_fname = os.path.join(save_dir, 'multi_lstm.h5')
log_fname = save_dir
model.save(save_fname)
model_score(model, train_x, train_y, val_x, val_y)
model_test_score(model, val_x, val_y)