#!/bin/bash
set -e

ckpt_dir="hdfs://xxx/test"
data_hdfs_path="hdfs://xxx/feat"
meta_path="hdfs://xxx/meta"
predict_interval=3

function Run() {
  cp template.json config.json
  sed -i "s?#ckpt_dir#?${ckpt_dir}?g" config.json
  
  cp template.py model.py
  sed -i "s?#train_hdfs_path#?$1?g" model.py
  sed -i "s?#predict_hdfs_path#?$2?g" model.py
  sed -i "s?#meta_path#?${meta_path}?g" model.py
  
  predict_hour=${2:0-2}
  if [ $[10#${predict_hour} % ${predict_interval}] -eq 0 ];then
      sed -i "s?#predict_flag#?True?g" model.py
      #echo ${predict_hour}
  else
      sed -i "s?#predict_flag#?False?g" model.py
  fi
}


function Batch() {
    batch_begin="20210101/03"
    batch_end="20210101/04"
	until [[ $batch_begin > $batch_end ]];do
		train_hdfs_path_list=${data_hdfs_path}/${batch_begin}
		batch_begin=$(date +%Y%m%d/%H -d "$(echo $batch_begin | sed 's/\// /') +1 hour")
		pred_hdfs_path_list=${data_hdfs_path}/${batch_begin}
        ### begin
        echo -n $(date '+[%Y/%m/%d %T]')
        echo " begin..."
		### train or predict
		echo -n $(date '+[%Y/%m/%d %T]')
		echo " train_data: ${train_hdfs_path_list}, training...."
		Run ${train_hdfs_path_list} ${pred_hdfs_path_list}
		submit_job.py --config config.json || continue
		
        echo -n $(date '+[%Y/%m/%d %T]')
		echo -e " end...\n"
	done
}

Batch

#!/bin/bash
set -e

start_date=20210101
end_date=20210101
predict_hour="00"

ckpt_dir="hdfs://xxx/test"
data_hdfs_path="hdfs://xxx/feat"
meta_path="hdfs://xxx/meta"

function Run() {
  cp template.json config.json
  sed -i "s?#ckpt_dir#?${ckpt_dir}?g" config.json
  
  cp template.py model.py
  sed -i "s?#train_hdfs_path#?$1?g" model.py
  sed -i "s?#predict_hdfs_path#?$2?g" model.py
  sed -i "s?#meta_path#?${meta_path}?g" model.py
 
  sed -i "s?#predict_flag#?True?g" model.py
}


cur_day=${start_date}
until [[ $cur_day > $end_date ]];do
  train_hdfs_path_list=${data_hdfs_path}/${cur_day}
  cur_day=$(date -d "${cur_day} +1 day" +%Y%m%d)
  pred_hdfs_path_list=${data_hdfs_path}/${cur_day}/${predict_hour}
  
  ### begin
  echo -n $(date '+[%Y/%m/%d %T]')
  echo " begin..."
  ### train or predict
  echo -n $(date '+[%Y/%m/%d %T]')
  echo " train_data: ${train_hdfs_path_list}, training...."
  Run ${train_hdfs_path_list} ${pred_hdfs_path_list}
  submit_job.py --config config.json || continue
  
  echo -n $(date '+[%Y/%m/%d %T]')
  echo -e " end...\n"
done