在数据挖掘任务中，特征工程占据相当大的工作量。最近做唯品会购物预测的比赛中，发现生成特征时候，使用list的append方法去连接不同样本的特征比用pandas的concat方法效率提高很大。然后再尝试用并行的方式对concat方法进行优化，发现无任何提高，不知道是不是姿势不对。 把代码粘贴一下，记录一下,tt是抽取的一个（100000，4）大小的dataframe进行运行时间的比较
1.append生成特征集
start = time()
X = []
for each_uid,group in tt.groupby(by=['uid']):
time_alpha = datetime.date(1900,3,20)
time_alpha2 = datetime.date(1900,4,1)
#df = pd.Series({'uid':each_uid})
user_activity = group.shape[0] #用户活跃度user_activity :总浏览次数
user_buyablity = group[group['action_type'] == 1].shape[0] #用户购买力：总购买数量
user_takerate = user_buyablity/user_activity #转化率
user_near_activity = 0 #最近10天的活跃度
for i in group['date']:
if(i.date() > time_alpha):
user_near_activity += 1
user_first_time = min(group['date']).date()#第一次活跃时间
user_last_time = max(group['date']).date()#最后一次活跃时间
user_timedalta = (user_last_time - user_first_time).days#用户总活跃时间差
user_near_timedelta = (time_alpha2 - user_last_time).days#用户最有一次活跃距离4.1的天数
X1 = [each_uid,user_activity,user_buyablity,user_takerate,user_near_activity,user_timedalta,user_near_timedelta]
X.append(X1)
names = ['each_uid','user_activity','user_buyablity','user_takerate','user_near_activity','user_timedalta','user_near_timedelta']
X = pd.DataFrame(X,columns=names)
stop = time()
print(str(stop-start) + "秒")
运行时间：42.0529999733秒
2.concat生成特征集
start = time()
X = pd.DataFrame()
for each_uid,group in tt.groupby(by=['uid']):
time_alpha = datetime.date(1900,3,20)
time_alpha2 = datetime.date(1900,4,1)
df = pd.Series({'uid':each_uid})
user_activity = group.shape[0] #用户活跃度user_activity :总浏览次数
user_buyablity = group[group['action_type'] == 1].shape[0] #用户购买力：总购买数量
user_takerate = user_buyablity/user_activity #转化率
user_near_activity = 0 #最近10天的活跃度
for i in group['date']:
if(i.date() > time_alpha):
user_near_activity += 1
user_first_time = min(group['date']).date()#第一次活跃时间
user_last_time = max(group['date']).date()#最后一次活跃时间
user_timedalta = (user_last_time - user_first_time).days#用户总活跃时间差
user_near_timedelta = (time_alpha2 - user_last_time).days#用户最有一次活跃距离4.1的天数
df['user_activity'] = user_activity
df['user_buyablity'] = user_buyablity
df['user_takerate'] = user_takerate
df['user_near_activity'] = user_near_activity
df['user_timedalta'] = user_timedalta
df['user_near_timedelta'] = user_near_timedelta
X = pd.concat([X,df.to_frame().T], axis=0)
stop = time()
print(str(stop-start) + "秒")
运行时间：284.087000132秒
3.并行处理
def applyParallel(dfGrouped,func):
with Parallel(n_jobs=32) as parallel:
retLst = parallel(delayed(func)(group) for name, group in dfGrouped )
return pd.concat(retLst, axis=0)
#user features user_num = 196030
#style 3:
def getUserFeatures(group):
time_alpha = datetime.date(1900,3,20)
time_alpha2 = datetime.date(1900,4,1)
df = pd.Series({'uid':group.iloc[0]['uid']})
user_activity = group.shape[0] #用户活跃度user_activity :总浏览次数
user_buyablity = group[group['action_type'] == 1].shape[0] #用户购买力：总购买数量
user_takerate = user_buyablity/user_activity #转化率
user_near_activity = 0 #最近10天的活跃度
for i in group['date']:
if(i.date() > time_alpha):
user_near_activity += 1
user_first_time = min(group['date']).date()#第一次活跃时间
user_last_time = max(group['date']).date()#最后一次活跃时间
user_timedalta = (user_last_time - user_first_time).days#用户总活跃时间差
user_near_timedelta = (time_alpha2 - user_last_time).days#用户最有一次活跃距离4.1的天数
df['user_activity'] = user_activity
df['user_buyablity'] = user_buyablity
df['user_takerate'] = user_takerate
df['user_near_activity'] = user_near_activity
df['user_timedalta'] = user_timedalta
df['user_near_timedelta'] = user_near_timedelta
return df.to_frame().T
X = applyParallel(tt.groupby(by=['uid']),getUserFeatures)
运行时间：卡住
