add xlearning train script

xudong.yang · xudong.yang · commit b9e1d046d8d5 · 2018-12-11T16:29:04.000+08:00
diff --git a/DIN/xlearning.sh b/DIN/xlearning.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+set -e -x 
+export HADOOP_HOME=/data2/hadoop-2.6.0-cdh5.4.0
+
+hadoop fs -rm -r -f hdfs://bigdata/tmp/bbdin/${bizdate}
+hadoop fs -rm -r -f hdfs://bigdata/tmp/bbdin-output/${bizdate}
+
+train_data_file=`hadoop fs -ls /user/xudong.yang/mainstream/samples | tail -n 7 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+test_data_file=`hadoop fs -ls /user/xudong.yang/mainstream/eval     | tail -n 7 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+
+/opt/beibei/xlearning/bin/xl-submit \
+   --app-type "tensorflow" \
+   --input-strategy "Placeholder" \
+   --app-name "mainstream_din_model" \
+   --board-logdir hdfs://bigdata/tmp/bbdin/${bizdate} \
+   --input ${train_data_file}#train_data\
+   --input ${test_data_file}#eval_data\
+   --files hdfs://bigdata/user/xudong.yang/din/train_bb.py,hdfs://bigdata/user/xudong.yang/din/bb_input_fn.py,hdfs://bigdata/user/xudong.yang/din/deep_interest_network.py \
+   --launch-cmd "python train_bb.py --learning_rate=0.01 --attention_hidden_units=16 --shuffle_buffer_size=25600 --save_checkpoints_steps=10000 --train_steps=200000 --batch_size=256\
+     --dropout_rate=0.5 --optimizer=Adagrad --train_data train_data --eval_data eval_data --model_dir hdfs://bigdata/tmp/bbdin/${bizdate} --output_model hdfs://bigdata/tmp/bbdin-output/${bizdate}" \
+   --worker-memory 12G \
+   --worker-num 10 \
+   --worker-cores 8 \
+   --ps-memory 10G \
+   --ps-num 1 \
+   --ps-cores 8 \
+   --queue default
+
+
+#hadoop fs -mkdir /user/xudong.yang/mainstream/bbdin_output/
+model_fold=`hadoop fs -ls hdfs://bigdata/tmp/bbdin-output/${bizdate} | tail -n 1 | awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+hadoop fs -rm -r -f /user/xudong.yang/mainstream/bbdin_output/*
+hadoop fs -cp ${model_fold}/* /user/xudong.yang/mainstream/bbdin_output
diff --git a/DUPN/xlearning.sh b/DUPN/xlearning.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+set -e -x 
+export HADOOP_HOME=/data2/hadoop-2.6.0-cdh5.4.0
+
+hadoop fs -rm -r -f hdfs://bigdata/tmp/dupn-share/${bizdate}
+hadoop fs -rm -r -f hdfs://bigdata/tmp/dupn-share-output/${bizdate}
+
+train_data_file=`hadoop fs -ls hdfs://bigdata/user/xudong.yang/bd_mainstream/new_samples/ | tail -n 7 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+test_data_file=`hadoop fs -ls hdfs://bigdata/user/xudong.yang/bd_mainstream/new_eval/     | tail -n 7 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+
+/opt/beibei/xlearning/bin/xl-submit \
+   --app-type "tensorflow" \
+   --input-strategy "Placeholder" \
+   --app-name "bd_mainstream_share_dupn_model" \
+   --board-logdir hdfs://bigdata/tmp/dupn-share/${bizdate} \
+   --input ${train_data_file}#train_data\
+   --input ${test_data_file}#eval_data\
+   --files hdfs://bigdata/user/xudong.yang/mainstream/model/dupn.py \
+   --launch-cmd "python dupn.py --hidden_units=512,256 --learning_rate=0.005 --shuffle_buffer_size=10000 --save_checkpoints_steps=10000 --train_steps=100000 --batch_size=256\
+     --train_data train_data --eval_data eval_data --model_dir hdfs://bigdata/tmp/dupn-share/${bizdate} --output_model hdfs://bigdata/tmp/dupn-share-output/${bizdate}" \
+   --worker-memory 15G \
+   --worker-num 5 \
+   --worker-cores 8 \
+   --ps-memory 5G \
+   --ps-num 1 \
+   --ps-cores 5 \
+   --queue default
+
+set +e
+hadoop fs -test -e /user/xudong.yang/mainstream/dupn_share_output/
+[ $? -ne 0 ] && hadoop fs -mkdir /user/xudong.yang/mainstream/dupn_share_output/
+model_fold=`hadoop fs -ls hdfs://bigdata/tmp/dupn-share-output/${bizdate} | awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+hadoop fs -rm -r -f /user/xudong.yang/mainstream/dupn_share_output/*
+hadoop fs -cp ${model_fold}/* /user/xudong.yang/mainstream/dupn_share_output
diff --git a/base-model/xlearning.sh b/base-model/xlearning.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+set -e -x 
+export GRPC_VERBOSITY="DEBUG"
+export HADOOP_HOME=/data2/hadoop-2.6.0-cdh5.4.0
+
+hadoop fs -rm -r -f hdfs://bigdata/tmp/base-model/${bizdate}
+hadoop fs -rm -r -f hdfs://bigdata/tmp/base-model-output/${bizdate}
+#hadoop fs -rm -f hdfs://bigdata/user/xudong.yang/mainstream/samples/pt=${bizdate}/_SUCCESS
+train_data_file=`hadoop fs -ls /user/xudong.yang/mainstream/samples | tail -n 7 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+test_data_file=`hadoop fs -ls /user/xudong.yang/mainstream/eval     | tail -n 7 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+
+/opt/beibei/xlearning/bin/xl-submit \
+   --app-type "tensorflow" \
+   --input-strategy "Placeholder" \
+   --app-name "mainstream_dnn_base_model" \
+   --board-logdir hdfs://bigdata/tmp/base-model/${bizdate} \
+   --input ${train_data_file}#train_data\
+   --input ${test_data_file}#eval_data\
+   --files hdfs://bigdata/user/xudong.yang/mainstream/model/base-model-weighted.py \
+   --launch-cmd "python base-model-weighted.py --save_checkpoints_steps=10000 --train_steps=200000 --batch_size=256 --train_data train_data --eval_data eval_data --model_dir hdfs://bigdata/tmp/base-model/${bizdate} --output_model hdfs://bigdata/tmp/base-model-output/${bizdate}" \
+   --worker-memory 10G \
+   --worker-num 6 \
+   --worker-cores 8 \
+   --ps-memory 3G \
+   --ps-num 1 \
+   --ps-cores 5 \
+   --queue default
+
+#hadoop fs -mkdir /user/xudong.yang/mainstream/model_output_v2/
+model_fold=`hadoop fs -ls hdfs://bigdata/tmp/base-model-output/${bizdate} | awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+hadoop fs -rm -r -f /user/xudong.yang/mainstream/model_output_v2/*
+hadoop fs -cp ${model_fold}/* /user/xudong.yang/mainstream/model_output_v2
diff --git a/deepFM/xlearning.sh b/deepFM/xlearning.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+set -e -x 
+export HADOOP_HOME=/data2/hadoop-2.6.0-cdh5.4.0
+
+hadoop fs -rm -r -f hdfs://bigdata/tmp/deepfm/${bizdate}
+hadoop fs -rm -r -f hdfs://bigdata/tmp/deepfm-output/${bizdate}
+
+train_data_file=`hadoop fs -ls /user/xudong.yang/mainstream/samples | tail -n 7 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+test_data_file=`hadoop fs -ls /user/xudong.yang/mainstream/eval     | tail -n 7 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+
+/opt/beibei/xlearning/bin/xl-submit \
+   --app-type "tensorflow" \
+   --input-strategy "Placeholder" \
+   --app-name "mainstream_deepfm_model" \
+   --board-logdir hdfs://bigdata/tmp/deepfm/${bizdate} \
+   --input ${train_data_file}#train_data\
+   --input ${test_data_file}#eval_data\
+   --files hdfs://bigdata/user/xudong.yang/deepfm/train_deepfm.py,hdfs://bigdata/user/xudong.yang/deepfm/deepfm_input_fn.py,hdfs://bigdata/user/xudong.yang/deepfm/deepfm.py \
+   --launch-cmd "python train_deepfm.py --learning_rate=0.01 --shuffle_buffer_size=30000 --save_checkpoints_steps=10000 --train_steps=200000 --batch_size=256\
+     --dropout_rate=0.5 --train_data train_data --eval_data eval_data --model_dir hdfs://bigdata/tmp/deepfm/${bizdate} --output_model hdfs://bigdata/tmp/deepfm-output/${bizdate}" \
+   --worker-memory 12G \
+   --worker-num 6 \
+   --worker-cores 8 \
+   --ps-memory 4G \
+   --ps-num 1 \
+   --ps-cores 5 \
+   --queue default
+
+set +e
+hadoop fs -test -e /user/xudong.yang/mainstream/deepfm_output/
+[ $? -ne 0 ] && hadoop fs -mkdir /user/xudong.yang/mainstream/deepfm_output/
+model_fold=`hadoop fs -ls hdfs://bigdata/tmp/deepfm-output/${bizdate} | tail -n 1 | awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+hadoop fs -rm -r -f /user/xudong.yang/mainstream/deepfm_output/*
+hadoop fs -cp ${model_fold}/* /user/xudong.yang/mainstream/deepfm_output
diff --git a/esmm/xlearning.sh b/esmm/xlearning.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+set -e -x 
+export HADOOP_HOME=/data2/hadoop-2.6.0-cdh5.4.0
+
+hadoop fs -rm -r -f hdfs://bigdata/tmp/esmm-model/${bizdate}
+hadoop fs -rm -r -f hdfs://bigdata/tmp/esmm-model-output/${bizdate}
+hadoop fs -rm -f hdfs://bigdata/user/xudong.yang/bd_mainstream/samples/pt=${bizdate}/_SUCCESS
+
+train_data_file=`hadoop fs -ls /user/xudong.yang/bd_mainstream/samples | tail -n 5 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+test_data_file=`hadoop fs -ls /user/xudong.yang/bd_mainstream/eval     | tail -n 5 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+
+/opt/beibei/xlearning/bin/xl-submit \
+   --app-type "tensorflow" \
+   --input-strategy "Placeholder" \
+   --app-name "bd_mainstream_esmm_model" \
+   --board-logdir hdfs://bigdata/tmp/esmm-model/${bizdate} \
+   --input ${train_data_file}#train_data\
+   --input ${test_data_file}#eval_data\
+   --files hdfs://bigdata/user/xudong.yang/mainstream/model/esmm.py \
+   --launch-cmd "python esmm.py --hidden_units=512,256 --learning_rate=0.005 --shuffle_buffer_size=300000 --save_checkpoints_steps=10000 --train_steps=100000 --batch_size=512\
+     --train_data train_data --eval_data eval_data --model_dir hdfs://bigdata/tmp/esmm-model/${bizdate} --output_model hdfs://bigdata/tmp/esmm-model-output/${bizdate}" \
+   --worker-memory 10G \
+   --worker-num 6 \
+   --worker-cores 8 \
+   --ps-memory 3G \
+   --ps-num 1 \
+   --ps-cores 5 \
+   --queue default
+
+#hadoop fs -mkdir /user/xudong.yang/mainstream/esmm_model_output/
+model_fold=`hadoop fs -ls hdfs://bigdata/tmp/esmm-model-output/${bizdate} | awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+hadoop fs -rm -r -f /user/xudong.yang/mainstream/esmm_model_output/*
+hadoop fs -cp ${model_fold}/* /user/xudong.yang/mainstream/esmm_model_output
diff --git a/esmm_ext/xlearning.sh b/esmm_ext/xlearning.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+set -e -x 
+export HADOOP_HOME=/data2/hadoop-2.6.0-cdh5.4.0
+
+hadoop fs -rm -r -f hdfs://bigdata/tmp/esmm-dcn/${bizdate}
+hadoop fs -rm -r -f hdfs://bigdata/tmp/esmm-dcn-output/${bizdate}
+
+train_data_file=`hadoop fs -ls /user/xudong.yang/bd_mainstream/samples | tail -n 5 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+test_data_file=`hadoop fs -ls /user/xudong.yang/bd_mainstream/eval     | tail -n 5 |awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+
+/opt/beibei/xlearning/bin/xl-submit \
+   --app-type "tensorflow" \
+   --input-strategy "Placeholder" \
+   --app-name "bd_mainstream_esmm_dcn" \
+   --board-logdir hdfs://bigdata/tmp/esmm-dcn/${bizdate} \
+   --input ${train_data_file}#train_data\
+   --input ${test_data_file}#eval_data\
+   --files hdfs://bigdata/user/xudong.yang/esmm/esmm.py,hdfs://bigdata/user/xudong.yang/esmm/train.py,hdfs://bigdata/user/xudong.yang/esmm/dcn_input_fn.py,hdfs://bigdata/user/xudong.yang/esmm/dcn_logit_fn.py,hdfs://bigdata/user/xudong.yang/esmm/din_logit_fn.py \
+   --launch-cmd "python train.py --hidden_units=256,256 --learning_rate=0.0005 --shuffle_buffer_size=12800 --save_checkpoints_steps=10000 --train_steps=10000 --batch_size=128\
+     --use_batch_norm=false --train_data train_data --eval_data eval_data --model_dir hdfs://bigdata/tmp/esmm-dcn/${bizdate} --output_model hdfs://bigdata/tmp/esmm-dcn-output/${bizdate}" \
+   --worker-memory 15G \
+   --worker-num 8 \
+   --worker-cores 8 \
+   --ps-memory 5G \
+   --ps-num 1 \
+   --ps-cores 5 \
+   --queue default
+
+set +e
+hadoop fs -test -e /user/xudong.yang/mainstream/esmm_dcn_output/
+[ $? -ne 0 ] && hadoop fs -mkdir /user/xudong.yang/mainstream/esmm_dcn_output/
+model_fold=`hadoop fs -ls hdfs://bigdata/tmp/esmm-dcn-output/${bizdate} | awk -F" " '{print $8}' | xargs | sed -e 's/ /,/g'`
+hadoop fs -rm -r -f /user/xudong.yang/mainstream/esmm_dcn_output/*
+hadoop fs -cp ${model_fold}/* /user/xudong.yang/mainstream/esmm_dcn_output
