本文旨在进行时间序列预测，采用seq2seq+attention的模型架构进行预测，数据集样式如下：
数据集样式
一、导入包和数据
我都tensorflow版本是2.7，keras版本2.7
import csv
import numpy as np
from keras import backend as K
from keras.models import Model
from keras.layers import LSTM, Dense, Activation, Lambda, RepeatVector, Input, Reshape, Concatenate, Dot
from sklearn.preprocessing import MinMaxScaler
import os
import errno
import math
def file_processing(file_path, encode=None):
data = []
with open(file_path, encoding=encode) as file:
rows = csv.reader(file, delimiter=",")
n_row = 0
for row in rows:
if n_row != 0:
#column -> 0: code, 1: date
for column in range(2, len(row)):
data[n_row - 1].append(float(row[column].strip()))
data.append([])
n_row += 1
del data[-1]
return np.array(data)
data = file_processing('20180504_50.csv')
二、数据集的处理
####归一化
def normalize_data(data, scaler, feature_len):
minmaxscaler = scaler.fit(data)
normalize_data = minmaxscaler.transform(data)
return normalize_data
###处理成时序模型所需要的样本
def load_data(data, time_step=20, after_day=1, validate_percent=0.67):
seq_length = time_step + after_day
result = []
for index in range(len(data) - seq_length + 1):
result.append(data[index: index + seq_length])
result = np.array(result)
print('total data: ', result.shape)
train_size = int(len(result) * validate_percent)
train = result[:train_size, :]
validate = result[train_size:, :]
x_train = train[:, :time_step]
y_train = train[:, time_step:]
x_validate = validate[:, :time_step]
y_validate = validate[:, time_step:]
return [x_train, y_train, x_validate, y_validate]
time_step = 20 ###处理时序特征的时候20个作为一组
feature_len = data.shape[1] ###数据中有几列数据需要预测
scaler = MinMaxScaler(feature_range=(0, 1)) ###归一化
validate_percent = 0.8 ###验证集训练集拆分比例2：8
after_day = 6 ####预测之后6天
# 数据归一化
data = normalize_data(data, scaler, feature_len)
# 以最后的历史样本作为测试集
x_test = data[-time_step:]
x_test = np.reshape(x_test, (1, x_test.shape[0], x_test.shape[1]))
# 得到训练集和验证集
x_train, y_train, x_validate, y_validate = load_data(
data, time_step=time_step, after_day=after_day, validate_percent=validate_percent)
###一组20个特征，标签维度是6, 5列，用一步一步推进的方式构造出1025条训练样本
#x_train.shape, y_train.shape: (1025, 20, 5) (1025, 6, 5)
##x_validate.shape, y_validate.shape: (257, 20, 5) (257, 6, 5)
数据处理样式如图：
时序数据处理样式
三、模型建立
使用seq2seq+attention方式构建时序预测模型。
def softmax(x, axis=1):
"""Softmax activation function.
# Arguments
x : Tensor.
axis: Integer, axis along which the softmax normalization is applied.
# Returns
Tensor, output of softmax transformation.
# Raises
ValueError: In case `dim(x) == 1`.
"""
ndim = K.ndim(x)
if ndim == 2:
return K.softmax(x)
elif ndim > 2:
e = K.exp(x - K.max(x, axis=axis, keepdims=True))
s = K.sum(e, axis=axis, keepdims=True)
return e / s
else:
raise ValueError('Cannot apply softmax to a tensor that is 1D')
def mean_squared_error(y_true, y_pred):
return K.mean(K.square(y_pred[:,:,3] - y_true[:,:,3]), axis=-1)
def one_step_attention(a, s_prev, repeator, concatenator, densor, activator, dotor):
##将s_prev复制
s_prev = repeator(s_prev)
### 拼接BiRNN隐层状态与s_prev
concat = concatenator([s_prev, a])
e = densor(concat)
alphas = activator(e)
# 加权得到Context Vector
context = dotor([alphas, a])
return context
def seq2seq_attention(feature_len=1, after_day=1, input_shape=(20, 1), time_step=20):
# Define the inputs of your model with a shape (Tx, feature)
X = Input(shape=input_shape)
###Decoder端LSTM的初始状态
s0 = Input(shape=(100, ), name='s0')
c0 = Input(shape=(100, ), name='c0')
##Decoder前一时刻解码的输出s
s = s0
##Eecoder输出的c
c = c0
# 模型输出列表，用来存储预测的结果
all_outputs = []
# Encoder: pre-attention LSTM
encoder = LSTM(units=100, return_state=False, return_sequences=True, name='encoder')
# Decoder: post-attention LSTM
decoder = LSTM(units=100, return_state=True, name='decoder')
# Output
decoder_output = Dense(units=feature_len, activation='linear', name='output')
model_output = Reshape((1, feature_len))
# Attention
repeator = RepeatVector(time_step)
concatenator = Concatenate(axis=-1)
densor = Dense(1, activation = "relu")
activator = Activation(softmax, name='attention_weights')
dotor = Dot(axes = 1)
encoder_outputs = encoder(X)
for t in range(after_day):
context = one_step_attention(encoder_outputs, s, repeator, concatenator, densor, activator, dotor)
a, s, c = decoder(context, initial_state=[s, c])
outputs = decoder_output(a)
outputs = model_output(outputs)
all_outputs.append(outputs)
all_outputs = Lambda(lambda x: K.concatenate(x, axis=1))(all_outputs)
model = Model(inputs=[X, s0, c0], outputs=all_outputs)
return model
####输入每条构建样本大小
input_shape = (time_step, feature_len)
model = seq2seq_attention(feature_len, after_day, input_shape, time_step)
model.compile(loss=mean_squared_error, optimizer='adam')
model.summary()
四、模型的训练和评估
batch_size = 64
epochs = 300
def save_model(model, model_name):
file_path = 'model_test/{}.h5'.format(model_name)
if not os.path.exists(os.path.dirname(file_path)):
try:
os.makedirs(os.path.dirname(file_path))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
model.save(file_path)
s0_train = np.zeros((x_train.shape[0],100))
c0_train = np.zeros((x_train.shape[0],100))
s0_validate = np.zeros((x_validate.shape[0],100))
c0_validate = np.zeros((x_validate.shape[0],100))
s0_test = np.zeros((x_test.shape[0],100))
c0_test = np.zeros((x_test.shape[0],100))
###模型的训练
history = model.fit(
[x_train, s0_train, c0_train], y_train,
batch_size=batch_size, epochs=epochs,
validation_data=([x_validate, s0_validate, c0_validate], y_validate))
print('-' * 100)
train_score = model.evaluate([x_train,s0_train,c0_train], y_train, batch_size=batch_size, verbose=0)
print('Train Score: %.8f MSE (%.8f RMSE)' % (train_score, math.sqrt(train_score)))
validate_score = model.evaluate([x_validate,s0_validate,c0_validate], y_validate, batch_size=batch_size, verbose=0)
print('Test Score: %.8f MSE (%.8f RMSE)' % (validate_score, math.sqrt(validate_score)))
model_class_name = 'model23'
save_model(model, model_name=model_class_name)
五、模型预测
train_predict = model.predict([x_train,s0_train,c0_train])
validate_predict = model.predict([x_validate,s0_validate,c0_validate])
test_predict = model.predict([x_test,s0_test,c0_test]) ####预测出的真实值
###反归一化 归一化的数据要转换回去
def inverse_normalize_data(data, scaler):
for i in range(len(data)):
data[i] = scaler.inverse_transform(data[i])
return data
train_predict = inverse_normalize_data(train_predict, scaler)
y_train = inverse_normalize_data(y_train, scaler)
validate_predict = inverse_normalize_data(validate_predict, scaler)
y_validate = inverse_normalize_data(y_validate, scaler)
test_predict = inverse_normalize_data(test_predict, scaler)
print(test_predict) ###接下来6天的预测结果
六、完整代码如下
import csv
import numpy as np
from keras import backend as K
from keras.models import Model
from keras.layers import LSTM, Dense, Activation, Lambda, RepeatVector, Input, Reshape, Concatenate, Dot
from sklearn.preprocessing import MinMaxScaler
import os
import errno
import math
def file_processing(file_path, encode=None):
data = []
with open(file_path, encoding=encode) as file:
rows = csv.reader(file, delimiter=",")
n_row = 0
for row in rows:
if n_row != 0:
#column -> 0: code, 1: date
for column in range(2, len(row)):
data[n_row - 1].append(float(row[column].strip()))
data.append([])
n_row += 1
del data[-1]
return np.array(data)
####归一化
def normalize_data(data, scaler, feature_len):
minmaxscaler = scaler.fit(data)
normalize_data = minmaxscaler.transform(data)
return normalize_data
###处理成时序模型所需要的样本
def load_data(data, time_step=20, after_day=1, validate_percent=0.67):
seq_length = time_step + after_day
result = []
for index in range(len(data) - seq_length + 1):
result.append(data[index: index + seq_length])
result = np.array(result)
print('total data: ', result.shape)
train_size = int(len(result) * validate_percent)
train = result[:train_size, :]
validate = result[train_size:, :]
x_train = train[:, :time_step]
y_train = train[:, time_step:]
x_validate = validate[:, :time_step]
y_validate = validate[:, time_step:]
return [x_train, y_train, x_validate, y_validate]
def softmax(x, axis=1):
"""Softmax activation function.
# Arguments
x : Tensor.
axis: Integer, axis along which the softmax normalization is applied.
# Returns
Tensor, output of softmax transformation.
# Raises
ValueError: In case `dim(x) == 1`.
"""
ndim = K.ndim(x)
if ndim == 2:
return K.softmax(x)
elif ndim > 2:
e = K.exp(x - K.max(x, axis=axis, keepdims=True))
s = K.sum(e, axis=axis, keepdims=True)
return e / s
else:
raise ValueError('Cannot apply softmax to a tensor that is 1D')
def mean_squared_error(y_true, y_pred):
return K.mean(K.square(y_pred[:,:,3] - y_true[:,:,3]), axis=-1)
def one_step_attention(a, s_prev, repeator, concatenator, densor, activator, dotor):
##将s_prev复制
s_prev = repeator(s_prev)
### 拼接BiRNN隐层状态与s_prev
concat = concatenator([s_prev, a])
e = densor(concat)
alphas = activator(e)
# 加权得到Context Vector
context = dotor([alphas, a])
return context
def seq2seq_attention(feature_len=1, after_day=1, input_shape=(20, 1), time_step=20):
# Define the inputs of your model with a shape (Tx, feature)
X = Input(shape=input_shape)
###Decoder端LSTM的初始状态
s0 = Input(shape=(100, ), name='s0')
c0 = Input(shape=(100, ), name='c0')
##Decoder前一时刻解码的输出s
s = s0
##Eecoder输出的c
c = c0
# 模型输出列表，用来存储预测的结果
all_outputs = []
# Encoder: pre-attention LSTM
encoder = LSTM(units=100, return_state=False, return_sequences=True, name='encoder')
# Decoder: post-attention LSTM
decoder = LSTM(units=100, return_state=True, name='decoder')
# Output
decoder_output = Dense(units=feature_len, activation='linear', name='output')
model_output = Reshape((1, feature_len))
# Attention
repeator = RepeatVector(time_step)
concatenator = Concatenate(axis=-1)
densor = Dense(1, activation = "relu")
activator = Activation(softmax, name='attention_weights')
dotor = Dot(axes = 1)
encoder_outputs = encoder(X)
for t in range(after_day):
context = one_step_attention(encoder_outputs, s, repeator, concatenator, densor, activator, dotor)
a, s, c = decoder(context, initial_state=[s, c])
outputs = decoder_output(a)
outputs = model_output(outputs)
all_outputs.append(outputs)
all_outputs = Lambda(lambda x: K.concatenate(x, axis=1))(all_outputs)
model = Model(inputs=[X, s0, c0], outputs=all_outputs)
return model
def save_model(model, model_name):
file_path = 'model_test/{}.h5'.format(model_name)
if not os.path.exists(os.path.dirname(file_path)):
try:
os.makedirs(os.path.dirname(file_path))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
model.save(file_path)
###反归一化
def inverse_normalize_data(data, scaler):
for i in range(len(data)):
data[i] = scaler.inverse_transform(data[i])
return data
if __name__ == '__main__':
data = file_processing('20180504_50.csv')
time_step = 20 ###处理时序特征的时候20个作为一组
feature_len = data.shape[1] ###数据中有几列数据需要预测
scaler = MinMaxScaler(feature_range=(0, 1)) ###归一化
validate_percent = 0.8 ###验证集训练集拆分比例2：8
after_day = 6 ####预测之后6天
# 数据归一化
data = normalize_data(data, scaler, feature_len)
# 以最后的历史样本作为测试集
x_test = data[-time_step:]
x_test = np.reshape(x_test, (1, x_test.shape[0], x_test.shape[1]))
# 得到训练集和验证集
x_train, y_train, x_validate, y_validate = load_data(
data, time_step=time_step, after_day=after_day, validate_percent=validate_percent)
###一组20个特征，6个标签,5列，用一步一步推进的方式构造出1025条训练样本
#x_train.shape, y_train.shape: (1025, 20, 5) (1025, 6, 5)
##x_validate.shape, y_validate.shape: (257, 20, 5) (257, 6, 5)
####输入每条构建样本大小
input_shape = (time_step, feature_len)
model = seq2seq_attention(feature_len, after_day, input_shape, time_step)
model.compile(loss=mean_squared_error, optimizer='adam')
model.summary()
batch_size = 64
epochs = 300
s0_train = np.zeros((x_train.shape[0],100))
c0_train = np.zeros((x_train.shape[0],100))
s0_validate = np.zeros((x_validate.shape[0],100))
c0_validate = np.zeros((x_validate.shape[0],100))
s0_test = np.zeros((x_test.shape[0],100))
c0_test = np.zeros((x_test.shape[0],100))
history = model.fit(
[x_train, s0_train, c0_train], y_train,
batch_size=batch_size, epochs=epochs,
validation_data=([x_validate, s0_validate, c0_validate], y_validate))
print('-' * 100)
train_score = model.evaluate([x_train,s0_train,c0_train], y_train, batch_size=batch_size, verbose=0)
print('Train Score: %.8f MSE (%.8f RMSE)' % (train_score, math.sqrt(train_score)))
validate_score = model.evaluate([x_validate,s0_validate,c0_validate], y_validate, batch_size=batch_size, verbose=0)
print('Test Score: %.8f MSE (%.8f RMSE)' % (validate_score, math.sqrt(validate_score)))
model_class_name = 'model22'
save_model(model, model_name=model_class_name)
train_predict = model.predict([x_train,s0_train,c0_train])
validate_predict = model.predict([x_validate,s0_validate,c0_validate])
test_predict = model.predict([x_test,s0_test,c0_test]) ####预测出的真实值
train_predict = inverse_normalize_data(train_predict, scaler)
y_train = inverse_normalize_data(y_train, scaler)
validate_predict = inverse_normalize_data(validate_predict, scaler)
y_validate = inverse_normalize_data(y_validate, scaler)
test_predict = inverse_normalize_data(test_predict, scaler)
print(test_predict)
七、总结
seq2seq（sequence to sequence）模型一般包括两个部分：编码器（encoder）和解码器（decoder），其中编码器主要是用于处理序列信息，解码器将这个信息处理出来得到一个输出的序列，这样的模型允许使用长度不同的输入和输出序列。而Attention将底层特征直接传递至网络的后层，可在一定程度上解决细节丢失和梯度消失的问题。
