#! /bin/bash
################################################################
#
#Usg:         get_atms.bash  yyyy-mm-dd
#Usg example: get_atms.bash  2012-04-09
#
#
#You do what: only need replace part in dir_local with your own:
# 
# /home/pub/external_data/rdr/npp_atms
#
#  --------->
#
#  YOUR_OWN_PATH_DEFINITION
# 
#History:
#
#04/10/2012 Wanchun Chen
# 
################################################################

if [[ $# -eq 0 ]] ; then
  echo "get_j01_atms.bash yyyy-mm-dd"
  exit
fi

ymd=$1
yyyymmdd=`echo $ymd | cut -c1-4,6-7,9-10`


#---- The only thing you need change
dir_local=/data/data248/pub/external_data/rdr/n20_atms/${ymd}
mkdir -p ${dir_local}
chmod g+w ${dir_local}

echo "files from SCDR:"
#list_gatmo=`/data/starfs1/bin/scdr-files -t GATMO ${ymd} 2> /dev/null`
list_gatmo=`/data/starfs1/bin/scdr-files -t GATMO -satid J01 ${ymd} 2>&1`
echo "list_gatmo scdr return value=" $?
files_gatmo=($list_gatmo)
nfile_gatmo=${#files_gatmo[@]}
echo "nfile_gatmo=${nfile_gatmo}"
for f in ${list_gatmo}; do
  cp ${f} ${dir_local}/
done

#list_tatms=`/data/starfs1/bin/scdr-files -t TATMS ${ymd} 2> /dev/null`
list_tatms=`/data/starfs1/bin/scdr-files -t TATMS -satid J01 ${ymd} 2>&1`
echo "list_tatms scdr return value=" $?
files_tatms=($list_tatms)
nfile_tatms=${#files_tatms[@]}
echo "nfile_tatms=${nfile_tatms}"
for f in ${list_tatms}; do
  cp ${f} ${dir_local}/
done

#list_satms=`/data/starfs1/bin/scdr-files -t SATMS ${ymd} 2> /dev/null`
list_satms=`/data/starfs1/bin/scdr-files -t SATMS -satid J01 ${ymd} 2>&1`
echo "list_satms scdr return value=" $?
files_satms=($list_satms)
nfile_satms=${#files_satms[@]}
echo "nfile_satms=${nfile_satms}"
for f in ${list_satms}; do
  cp ${f} ${dir_local}/
done


#---- remove non-matching file names to fix cross-day file name problem

find ${dir_local} -type f ! -name "*_j01_d${yyyymmdd}_t*" -exec rm -f {} \;


# check and remove any duplicated granules and only keep the last one
#echo "check and remove any duplicated granules and only keep the last one for 3 sets"

echo "files copied locally:"
cd ${dir_local}
list_geo=`ls -1 GATMO_* 2> /dev/null`
files_geo=($list_geo)
nfile_geo=${#files_geo[@]}
echo "nfile_geo=${nfile_geo}"
for (( ifile=0; ifile<${nfile_geo}; ifile++ )) ; do
   file_geo=${files_geo[$ifile]}
   token=`echo ${file_geo} | cut -f1-6 -d_`
   nfile=`ls -1 ${token}_c*.h5 | wc -l`
   if [[ ${nfile} -gt 1 ]] ; then
     list=`ls -1 ${token}_c*.h5`
     files=($list)
     for (( i=0; i<${nfile}-1; i++ )) ; do
       rm -f ${files[$i]}
     done
   fi
done

cd ${dir_local}
list_tdr=`ls -1 TATMS_* 2> /dev/null`
files_tdr=($list_tdr)
nfile_tdr=${#files_tdr[@]}
echo "nfile_tdr=${nfile_tdr}"
for (( ifile=0; ifile<${nfile_tdr}; ifile++ )) ; do
   file_tdr=${files_tdr[$ifile]}
   token=`echo ${file_tdr} | cut -f1-6 -d_`
   nfile=`ls -1 ${token}_c*.h5 | wc -l`
   if [[ ${nfile} -gt 1 ]] ; then
     list=`ls -1 ${token}_c*.h5`
     files=($list)
     for (( i=0; i<${nfile}-1; i++ )) ; do
       rm -f ${files[$i]}
     done
   fi
done

cd ${dir_local}
list_sdr=`ls -1 SATMS_* 2> /dev/null`
files_sdr=($list_sdr)
nfile_sdr=${#files_sdr[@]}
echo "nfile_sdr=${nfile_sdr}"
for (( ifile=0; ifile<${nfile_sdr}; ifile++ )) ; do
   file_sdr=${files_sdr[$ifile]}
   token=`echo ${file_sdr} | cut -f1-6 -d_`
   nfile=`ls -1 ${token}_c*.h5 | wc -l`
   if [[ ${nfile} -gt 1 ]] ; then
     list=`ls -1 ${token}_c*.h5`
     files=($list)
     for (( i=0; i<${nfile}-1; i++ )) ; do
       rm -f ${files[$i]}
     done
   fi
done


#echo "remove any unmatched granule and only keep matched ones with one another"
echo "removing unmatched granules:"

# for each GEO file, if no matched SDR or TDR file, we remove this GEO file
list_gat=`ls -1 GATMO_* 2> /dev/null`
files_gat=($list_gat)
nfile_gat=${#files_gat[@]}
echo "nfile_gat=${nfile_gat}"
for (( ifile=0; ifile<${nfile_gat}; ifile++ )) ; do
   file_gat=${files_gat[$ifile]}
   token=`echo ${file_gat} | cut -f2-6 -d_`
   file_tdr=`ls -1 TATMS_${token}_c*.h5 2> /dev/null`
   file_sdr=`ls -1 SATMS_${token}_c*.h5 2> /dev/null`
   if [[ -z ${file_tdr} || -z ${file_sdr} ]] ; then
     #echo "NO matched TATMS or SATMS files for ${file_gat}"
     rm -f ${file_gat}
   fi
done

# for each TDR file, if no matched GEO or SDR file, we remove this TDR file
cd ${dir_local}
list_tdr=`ls -1 TATMS_* 2> /dev/null`
files_tdr=($list_tdr)
nfile_tdr=${#files_tdr[@]}
echo "nfile_tdr=${nfile_tdr}"
for (( ifile=0; ifile<${nfile_tdr}; ifile++ )) ; do
   file_tdr=${files_tdr[$ifile]}
   token=`echo ${file_tdr} | cut -f2-6 -d_`
   file_gat=`ls -1 GATMO_${token}_c*.h5 2> /dev/null`
   file_sdr=`ls -1 SATMS_${token}_c*.h5 2> /dev/null`
   if [[ -z ${file_gat} || -z ${file_sdr} ]] ; then
     #echo "NO matched GATMO or SATMS file for ${file_tdr}"
     rm -f ${file_tdr}
   fi
done

# for each SDR file, if no matched GEO or TDR file, we remove this SDR file
cd ${dir_local}
list_sdr=`ls -1 SATMS_* 2> /dev/null`
files_sdr=($list_sdr)
nfile_sdr=${#files_sdr[@]}
echo "nfile_sdr=${nfile_sdr}"
for (( ifile=0; ifile<${nfile_sdr}; ifile++ )) ; do
   file_sdr=${files_sdr[$ifile]}
   token=`echo ${file_sdr} | cut -f2-6 -d_`
   file_gat=`ls -1 GATMO_${token}_c*.h5 2> /dev/null`
   file_tdr=`ls -1 TATMS_${token}_c*.h5 2> /dev/null`
   if [[ -z ${file_gat} || -z ${file_tdr} ]] ; then
     #echo "NO matched GATMO or TATMS file for ${file_sdr}"
     rm -f ${file_sdr}
   fi
done


#echo "after cleanup, 3 set of files should have equal numbers."
cd ${dir_local}

CNT_GATMO=`ls -1 GATMO_* | wc -l`
CNT_TATMS=`ls -1 TATMS_* | wc -l`
CNT_SATMS=`ls -1 SATMS_* | wc -l`

if [[ ${CNT_GATMO} -ne ${CNT_TATMS} || ${CNT_GATMO} -ne ${CNT_SATMS} || ${CNT_TATMS} -ne ${CNT_SATMS} ]] ; then
  #echo "ERROR: GATMO,TATMS and SATMS file numbers are not equal. STOP processing."
  #exit 1
  echo "-1"
elif [[ ${CNT_GATMO} -eq 0 ]] ; then
  #echo "ERROR: NO files. STOP processing."
  echo 0
else
  echo ${CNT_GATMO}
fi

chmod 774 ${dir_local}/*