- graph_embedding.jl_jingyan_query_related_top_query_detailed
取前两页， 保证正样本均衡， 1 ： 2采样 - 修改inference_v4
drop table hs_dssm_result_0;
yes
create table hs_dssm_result_0 like hs_tmp_71;
truncate tabel hs_dssm_result_0;
pai -name tensorflow140 -Dscript="file:///home/hengsong/origin_deep_cluster_odps_8.tar.gz" -DentryFile="inference_v4.py" -Dcluster='{"worker":{"count":1, "cpu":200, "memory":4000}, "ps":{"count":1, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/hs_dssm_inference_2" -Doutputs="odps://graph_embedding/tables/hs_dssm_result_0" -DcheckpointDir="oss://bucket-automl/hengsong/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=3e-4 --batch_size=1024 --is_save_model=False --attention_type=1 --num_epochs=1 --ckpt=hs_ugc_video.ckpt-3" -DuseSparseClusterSchema=True;
drop table hs_tmp_97;
yes
create table hs_tmp_97 as
select c., d.words as video_words from
(select a., b.words as query_words from
(select video_id as query_id, query_id as video_id, score, query_emb, video_emb from hs_dssm_result_0 where score > 0.5)a join (select * from hs_tmp_dssm_inf_querys)b on a.query_id == b.id)c join (select * from hs_tmp_dssm_inf_titles)d on c.video_id == d.id;
0.000000,0.000000,0.000000,273.329590,0.000000,0.000711,0.851089,162.811890,0.003108,0.000000,0.000000,0.000000,0.000000,300.079468,13.062472,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,35.827782,0.000000,0.000000,0.000012,0.000000,23.806574,4.368169,0.000000,0.000000,0.000052,0.000000,150.262299,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,43.310566,0.000000,0.000000,49.213623,0.000000,0.000000,0.000000,0.000000,0.000000,16.064854,38.540794,0.000000,0.000000,0.000000,0.000228,0.000000,0.000000
0.000000,0.000000,0.000000,28.452734,0.000000,0.075494,1.052082,0.000000,0.712099,0.000000,0.000000,0.000000,0.000000,7.956234,0.223112,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,4.467260,0.000000,0.000000,0.000000,0.000000,0.000000,0.548727,0.000000,0.000000,0.001843,0.000000,30.187546,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,2.689540,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,1.655775,0.000000,0.000000
- 使用训练集进行inference
drop table hs_train_data_dssm_4;
yes
create table hs_train_data_dssm_4 as select item_id as video_id, title_mainse_ws as video_ws, index as query_id, se_keyword_mainse_ws as query_ws, label from hs_train_data_dssm_2 limit 100000;
truncate tabel hs_dssm_result_3;
pai -name tensorflow140 -Dscript="file:///home/hengsong/origin_deep_cluster_odps_8.tar.gz" -DentryFile="inference_v4.py" -Dcluster='{"worker":{"count":1, "cpu":200, "memory":4000}, "ps":{"count":1, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/hs_train_data_dssm_4" -Doutputs="odps://graph_embedding/tables/hs_dssm_result_3" -DcheckpointDir="oss://bucket-automl/hengsong/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=3e-4 --batch_size=1024 --is_save_model=False --attention_type=1 --num_epochs=1 --ckpt=hs_ugc_video.ckpt-2" -DuseSparseClusterSchema=True;
drop table hs_tmp_97;
yes
create table hs_tmp_97 as
select c., d.words as video_words from
(select a., b.words as query_words from
(select video_id as query_id, query_id as video_id, score from hs_dssm_result_3 where score > 0.5)a join (select * from hs_tmp_dssm_inf_querys)b on a.query_id == b.id)c join (select * from hs_tmp_dssm_inf_titles)d on c.video_id == d.id;
drop table hs_tmp_99;
yes
create table hs_tmp_99 as
select a.*, b.label, b.query_ws, b.video_ws from
(select video_id as query_id, query_id as video_id, score from hs_dssm_result_3)a join (select video_id, query_id, label, query_ws, video_ws from hs_train_data_dssm_4)b on a.video_id == b.video_id and a.query_id == b.query_id;
