#!/bin/bash ################################################################################################################# # # Description: # This is the GUI generated bash script used to run the MIRS testbed. # # Record of revisions: # Date Ver. Programmer Description of change # ============ ===== ======================= ============================================ # 09/03/2005 v0 Sid-Ahmed Boukabara Original script created # (NOAA/NESDIS/ORA/IMSG) # # 02/20/2006 v1 Ninghai Sun Modify script for operational testbed # (NOAA/NESDIS/ORA/IMSG) # # 03/20/2006 v2 Ninghai Sun Modify script to compromise to SSM/IS # (NOAA/NESDIS/ORA/IMSG) Change the way to find GDAS data to standard # # 03/31/2006 v3 Sid Ahmed Boukabara Changes related to : # (1) new footprint-matching, # (2) addition of new covariance matrix # (3) sensor ID (to distinguish sensor-dependent classifiers), # (4) flexible handling of scanlines shift (mhs vs amsu) # (5) bias correction method (bias removal or slope/intercept) # (6) Added threshold checking of relative humidity # 05/31/2006 v4 Sid Ahmed Boukabara Changes related to directory structure more in line with oper. # # # 12/19/2006 v5 Sid Ahmed Boukabara Added the capability to run the scripts in # (IMSG@NOAA/NESDIS/STAR) orbital mode in addition to daily mode. # # 12/22/2006 v6 Sid Ahmed Boukabara Major changes to make the script as simple as possible to # maintain and ultimately to be generated through the JAVA GUI. # # 12/28/2006 v7 Sid Ahmed Boukabara Extensive revision to make all functions general. # (IMSG@NOAA/NESDIS/STAR) Scripts to be generated automatically through a Java GUI. # This removes the need to maintain as many scripts as sensors. # 03/10/2010 v8 Wanchun Chen Many functions previously in script are moved into library. # (QSS/PSGS/DELL) # 11/04/2016 v9 Junye Chen Modified GUI to generate JPSS-1(N20)/ATMS enabled scripts. # (ESSIC/UMD@NOAA/NESDIS/STAR) # # ################################################################################################################# #------Store arguments & set starting date script=$0 oldargs=("$@") nargs=$# sdate=`date` MIRS_ROOT=`grep ^MIRS_ROOT ../setup/paths | cut -f2 -d '=' | sed -e 's/^[ ]*//'` HDF4LIB=`grep HDF4LIB ../setup/paths | cut -f2 -d '=' | sed -e 's/^[ ]*//'` HDF5LIB=`grep HDF5LIB ../setup/paths | cut -f2 -d '=' | sed -e 's/^[ ]*//'` HDF5BIN=`grep HDF5BIN ../setup/paths | cut -f2 -d '=' | sed -e 's/^[ ]*//'` HDFEOSLIB=`grep HDFEOSLIB ../setup/paths | cut -f2 -d '=' | sed -e 's/^[ ]*//'` SZIPLIB=`grep SZIPLIB ../setup/paths | cut -f2 -d '=' | sed -e 's/^[ ]*//'` ZLIBLIB=`grep ZLIBLIB ../setup/paths | cut -f2 -d '=' | sed -e 's/^[ ]*//'` NETCDF4LIB=`grep NETCDF4LIB ../setup/paths | cut -f2 -d '=' | sed -e 's/^[ ]*//'` #-----Include libraries and setup Info . ../scripts/script_functions.bash . ../setup/n18_pcf_daily_ECMWF.bash # Add more colon-separated shared libraries here: export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HDF4LIB:$HDFEOSLIB:$SZIPLIB:$ZLIBLIB:$HDF5LIB:$NETCDF4LIB # set stack size to unlimited ulimit -s unlimited displayVerif ${script} ${logFile} ${processMode} ${sensorId}\ ${outFMAccuracy} ${prefixFMAccuracy} ${nProfs2Retr} ${nProfs2Fwd}\ ${addDeviceNoise} ${monitorIterative} ${nAttempts} ${extBkgAtmUse} \ ${externalDataAvailable} ${monitorRetrieval} ${monitorFwd} ${geoLimit}\ ${minLat} ${maxLat} ${minLon} ${maxLon} ${maxDaysArchived} ${nDaysBack}\ ${tdrFormat} ${cend} ${dayUsed4Bias} ${dayUsed4Alg} ${nOrbits2Process}\ ${gifDensity} ${externalDataSrc} ${fmType} ${biasComputeMethod} \ ${nChoppedFilesPerOrbit} ${retrOnOrbitOrSubOrbit} ${retrOnWhichSDR}\ ${fwdMatrix2Use} ${sweClimoUse} ${makeOrNot} ${useCPU} extResol=`determineSpatialResol ${satId} ${fmType}` version='9999' if [[ -s ${rootPath}/version.txt ]] ; then version=`cat ${rootPath}/version.txt` fi os=`uname -s` #Linux g95 and gfortran is the same as AIX xlf90/95(stream,unformatted) #Linux ifort is (sequential,binary) accessStr='sequential' formStr='binary' if [[ ${os} == 'Linux' ]] ; then accessStr='sequential' formStr='binary' elif [[ ${os} == 'AIX' ]] ; then accessStr='stream' formStr='unformatted' fi gdasData=1 ecmwfData=2 gfsData=3 #----Construct date extension(s) for building directories if [[ ${nargs} -eq 0 ]] ; then extensions=`DetermineYesterdExtAndAlanysExt ${nDaysBack}` date=$(echo ${extensions}|cut -c1-10) rdrSensor1Dir=${rdrSensor1Path}/${date} rdrSensor2Dir=${rdrSensor2Path}/${date} elif [[ ${nargs} -eq 1 ]] ; then #----Check 3 different paths (absolute/relative/a nude dir name) indx_slash=`echo ${oldargs} | awk -v slash="/" '{printf index($1,slash)}'` if [[ ${indx_slash} -eq 1 ]] ; then # =1: absolute path inputPath=${oldargs} elif [[ ${indx_slash} -gt 1 ]] ; then # >1: relative path inputPath=$(readlink -f ${oldargs}) # to get absolute full path else # <1: a nude dir name with no slash in it inputPath=${rdrSensor1Path}/${oldargs} fi date=`getDateFromFile ${satId} ${inputPath}` echo ${satId} echo ${inputPath} if [[ ${date} == 'xxxx-xx-xx' ]] ; then # something wrong op_msg "Error: date $date is incorrect" exit 1 fi rdrSensor1Dir=${inputPath} rdrSensor2Dir=${inputPath} extensions=`DetermineExtAndAlanysExtFromArgument ${date}` else ErrMessDue2UsageInDailyMode fi rdirExt=$(echo ${extensions}|cut -c1-10) rdirAnalysExt=$(echo ${extensions}|cut -c12-21) rdirNextAnalysExt=$(echo ${extensions}|cut -c23-32) #----set the generic extension fileExt=`determineFileExt ${rdirExt}` fileAnalysExt=`determineFileExt ${rdirAnalysExt}` orbitInfo=Dummy #----Names of directories for both orbital and Daily processing tdrSensor1Dir=${tdrSensor1Path}/${rdirExt} tdrSensor2Dir=${tdrSensor2Path}/${rdirExt} sdrSensor1Dir=${sdrSensor1Path}/${rdirExt} sdrSensor2Dir=${sdrSensor2Path}/${rdirExt} fmsdrDir=${fmsdrPath}/${rdirExt} fmsdrChoppDir=${choppPath}/${rdirExt} edrDir=${edrPath}/${rdirExt} depDir=${depPath}/${rdirExt} sfrDir=${sfrPath}/${rdirExt} gridDir=${gridPath}/${rdirExt} ncDir=${ncPath}/${rdirExt} figsDir=${figsPath}/${rdirExt} regressRetrDir=${regressRetrPath}/${rdirExt} prepSweClimoDir=${prepSweClimoPath}/${rdirExt} ffCollocDir=${ffCollocPath}/${rdirExt} imsCollocDir=${imsCollocPath}/${rdirExt} orbitMonPath=${perfsMonitorPath}/orbitmon/ #----control file and input file identifiers depending on processing mode if [[ ${processMode} -eq 0 ]] ; then identifier=${orbitInfo} else identifier=${rdirExt} fi #----Control file rdr2tdrSensor1ControlFile=${rdr2tdrSensor1ControlFile}_${identifier}.in rdr2tdrSensor2ControlFile=${rdr2tdrSensor2ControlFile}_${identifier}.in mergeNedtControlFile=${mergeNedtControlFile}_${identifier}.in tdr2sdrSensor1ControlFile=${tdr2sdrSensor1ControlFile}_${identifier}.in tdr2sdrSensor2ControlFile=${tdr2sdrSensor2ControlFile}_${identifier}.in fmControlFile=${fmControlFile}_${identifier}.in modifyNedtControlFile=${modifyNedtControlFile}_${identifier}.in fmsdr2edrControlFile=${fmsdr2edrControlFile}_${identifier}.in grid2nwpControlFile=${grid2nwpControlFile}_${identifier}.in fwdControlFile=${fwdControlFile}_${identifier}.in regressControlFile=${regressControlFile}_${identifier}.in prepSweClimoControlFile=${prepSweClimoControlFile}_${identifier}.in ffCollocControlFile=${ffCollocControlFile}_${identifier}.in imsCollocControlFile=${imsCollocControlFile}_${identifier}.in choppControlFile=${choppControlFile}_${identifier}.in mergeEdrControlFile=${mergeEdrControlFile}_${identifier}.in vippControlFile=${vippControlFile}_${identifier}.in sfrControlFile=${sfrControlFile}_${identifier}.in sfr2depControlFile=${sfr2depControlFile}_${identifier}.in gridControlFile=${gridControlFile}_${identifier}.in nwpGridControlFile=${nwpGridControlFile}_${identifier}.in fwdGridControlFile=${fwdGridControlFile}_${identifier}.in biasGridControlFile=${biasGridControlFile}_${identifier}.in biasCompuControlFile=${biasCompuControlFile}_${identifier}.in biasVerifControlFile=${biasVerifControlFile}_${identifier}.in regressGenControlFile=${regressGenControlFile}_${identifier}.in figsGenControlFile=${figsGenControlFile}_${identifier}.in #---- Input file list rdrSensor1List=${rdrSensor1List}_${identifier}.list rdrSensor2List=${rdrSensor2List}_${identifier}.list tdrSensor1List=${tdrSensor1List}_${identifier}.list tdrSensor2List=${tdrSensor2List}_${identifier}.list sdrSensor1List=${sdrSensor1List}_${identifier}.list sdrSensor2List=${sdrSensor2List}_${identifier}.list fmsdrList=${fmsdrList}_${identifier}.list fmsdr4ChoppList=${fmsdr4ChoppList}_${identifier}.list fmsdr4NwpList=${fmsdr4NwpList}_${identifier}.list fmsdr4BiasList=${fmsdr4BiasList}_${identifier}.list fmsdr4RegressList=${fmsdr4RegressList}_${identifier}.list fmsdr4ApplyRegressList=${fmsdr4ApplyRegressList}_${identifier}.list fmsdr4PrepSweList=${fmsdr4PrepSweList}_${identifier}.list fmsdr4ffCollocList=${fmsdr4ffCollocList}_${identifier}.list fmsdr4SfrList=${fmsdr4SfrList}_${identifier}.list fmsdr4imsCollocList=${fmsdr4imsCollocList}_${identifier}.list imsColloc4nwpList=${imsColloc4nwpList}_${identifier}.list imsColloc4retrList=${imsColloc4retrList}_${identifier}.list prepSweClimo4VippList=${prepSweClimo4VippList}_${identifier}.list ffColloc4VippList=${ffColloc4VippList}_${identifier}.list edrList=${edrList}_${identifier}.list edr4BiasList=${edr4BiasList}_${identifier}.list dep4BiasList=${dep4BiasList}_${identifier}.list edr4MergeList=${edr4MergeList}_${identifier}.list depList=${depList}_${identifier}.list sfrList=${sfrList}_${identifier}.list gridNwpSfrList=${gridNwpSfrList}_${identifier}.list nedtList=${nedtList}_${identifier}.list nedtSensor1List=${nedtSensor1List}_${identifier}.list nedtSensor2List=${nedtSensor2List}_${identifier}.list gridSfcNwpAnalysList=${gridSfcNwpAnalysList}_${identifier}.list gridAtmNwpAnalysList=${gridAtmNwpAnalysList}_${identifier}.list nwpAnalysList=${nwpAnalysList}_${identifier}.list nwpAnalysRetrList=${nwpAnalysRetrList}_${identifier}.list nwpAnalys4BiasList=${nwpAnalys4BiasList}_${identifier}.list nwpAnalys4RegressList=${nwpAnalys4RegressList}_${identifier}.list fwdAnalys4BiasList=${fwdAnalys4BiasList}_${identifier}.list #----Names of directories exclusively for Daily processing figs4BiasDir=${perfsMonitorPath}/${rdirAnalysExt} qcCheckDir=${perfsMonitorPath}/${rdirExt} figs4RegressDir=${perfsMonitorPath}/${rdirAnalysExt} nwpAnalysDir=${nwpAnalysPath}/${rdirAnalysExt} fwdAnalysDir=${fwdAnalysPath}/${rdirAnalysExt} fmsdr4BiasDir=${fmsdrPath}/${rdirAnalysExt} edr4BiasDir=${edrPath}/${rdirAnalysExt} dep4BiasDir=${depPath}/${rdirAnalysExt} grid4BiasDir=${gridPath}/${rdirAnalysExt} #----Names of the files dynamically generated sensor1Nedt=${nedtSensor1Path}/${satId}_${sensor1}_nedt_${fileExt}.dat sensor2Nedt=${nedtSensor2Path}/${satId}_${sensor2}_nedt_${fileExt}.dat nedtExt=${fileExt} if [[ ${satId} == "f16" || ${satId} == "aqua" || ${satId} == "gcomw1" || ${satId} == "f17" || ${satId} == "f18" || ${satId} == "f19" || ${satId} == "fy3ri" || ${satId} == "trmm" || ${satId} == "gpm" || ${satId} == "mtma" || ${satId} == "mtsa" ]] ; then nedtBefFMFile=${nedtSensor1Path}/${satId}_${sensor1}_nedt_${fileExt}_befFM.dat nedtAftFMFile=${nedtSensor1Path}/${satId}_${sensor1}_nedt_${fileExt}_aftFM.dat nedtDir=${nedtSensor1Path}/${rdirExt} elif [[ ${satId} == "npp" || ${satId} == "n20" || ${satId} == "n21" || ${satId} == "metopSGA1" ]]; then nedtExt=`determineNedtExt ${satId} ${rdrSensor1Dir} ${rdrType}` nedtBefFMFile=${nedtSensor1Path}/${satId}_${sensor1}_nedt_${nedtExt}_befFM.dat nedtAftFMFile=${nedtSensor1Path}/${satId}_${sensor1}_nedt_${nedtExt}_aftFM.dat nedtDir=${nedtSensor1Path}/${rdirExt} else nedtBefFMFile=${nedtSensor1Sensor2Path}/${satId}_${sensor1}_${sensor2}_nedt_${fileExt}_befFM.dat nedtAftFMFile=${nedtSensor1Sensor2Path}/${satId}_${sensor1}_${sensor2}_nedt_${fileExt}_aftFM.dat nedtDir=${nedtSensor1Sensor2Path}/${rdirExt} fi sensor1Wt=${nedtSensor1Path}/${satId}_${sensor1}_wt_${fileExt}.dat sensor2Wt=${nedtSensor2Path}/${satId}_${sensor2}_wt_${fileExt}.dat sensor1Sensor2Wt=${nedtSensor1Sensor2Path}/${satId}_${sensor1}_${sensor2}_wt_${fileExt}.dat modelErrFile=${biasPath}/ModelErrFile_${satId}_${fileExt}.dat #Model err file that will be generated if [[ ${satId} == "npp" || ${satId} == "n20" || ${satId} == "n21" || ${satId} == "metopSGA1" ]]; then modelErrFile=${biasPath}/ModelErrFile_${satId}_${nedtExt}.dat fi logFile=${logFile}_${fileExt}.dat #----Names of files dynamically generated only for daily processing biasFile=${biasPath}/biasCorrec_${satId}_${fileAnalysExt}.dat #bias file that will be generated if [[ ${satId} == "npp" || ${satId} == "n20" || ${satId} == "n21" || ${satId} == "metopSGA1" ]]; then biasFile=${biasPath}/biasCorrec_${satId}_${nedtExt}.dat fi biasCheckFile=${biasPath}/biasAfterCorr_${satId}_${fileAnalysExt}.dat #bias residual file #----EDR/DEP/NWP grid/p2p products list edrGridStr="angle,chisq,em,nattempt,niter,psfc,qc,scanday,scanpos,sfc,tbf,tbu,tbc,tbl,temp,tskin,wv,clwp,rainp,graupelp,pressure" edrP2PStr="em,scanpos,sfc,tbu,tskin,wspd,temp,psfc,wv" depGridStr="clw,gs,iwp,lwp,rr,rwp,sfc2,sice,sicefy,sicemy,snow,swe,tpw,sfr,sfrprob" depP2PStr="angle,clw,iwp,lat,lon,lwp,rr,rwp,sfc2,swe,tpw" nwpGridStr="angle,chisq,clw,em,iwp,nattempt,niter,psfc,rr,scanday,scanpos,sfc,swe,tbf,tbu,tbc,tbl,temp,tpw,tskin,wv,clwp,rainp,graupelp,pressure" nwpP2PStr="clw,em,iwp,lwp,rwp,sfc,swe,tbu,tpw,tskin,temp,psfc,wv" if [[ ${satId} == "f16" || ${satId} == "f17" || ${satId} == "f18" || ${satId} == "f19" || ${satId} == "trmm" || ${satId} == "gpm" || ${satId} == "mtma" || ${satId} == "mtsa" ]] ; then depGridStr="clw,gs,iwp,lwp,rr,rwp,sfc2,sice,sicefy,sicemy,snow,swe,tpw,wspd" depP2PStr="angle,clw,iwp,lat,lon,lwp,rr,rwp,sfc2,swe,tpw,wspd" nwpGridStr="angle,chisq,clw,em,iwp,nattempt,niter,psfc,rr,scanday,scanpos,sfc,swe,tbf,tbu,tbc,tbl,temp,tpw,tskin,wspd,wv,clwp,rainp,graupelp,pressure" nwpP2PStr="clw,em,iwp,lwp,rwp,sfc,swe,tbu,tpw,tskin,temp,wspd,psfc,wv" fi #---- fwd model error matrix modelErrFile1ToUse=${biasPath}/ModelErrFile_${satId}.dat modelErrFile2ToUse=${biasPath}/ModelErrFile_${satId}.dat #---Check existence of the directories. If negative, make them DirGen ${tdrSensor1Dir} "TDR-${sensor1}" if [[ $sensor2 != "dummy" ]] ; then DirGen ${tdrSensor2Dir} "TDR-${sensor2}" fi DirGen ${sdrSensor1Dir} "SDR-${sensor1}" if [[ $sensor2 != "dummy" ]] ; then DirGen ${sdrSensor2Dir} "SDR-${sensor2}" fi if [[ ${satId} == "npp" || ${satId} == "n20" || ${satId} == "n21" || ${satId} == "metopSGA1" ]] ; then DirGen ${nedtDir} "NEDT" fi DirGen ${fmsdrDir} "FM-SDR" DirGen ${fmsdrChoppDir} "CHOPPED FMSDRs" DirGen ${edrDir} "EDR" DirGen ${depDir} "DEP" DirGen ${gridDir} "GRID" DirGen ${ncDir} "NETCDF4" DirGen ${figsDir} "FIGS" DirGen ${sfrDir} "SFR" DirGen ${figs4BiasDir} "FIGS" DirGen ${grid4BiasDir} "GRID-BIAS" DirGen ${qcCheckDir} "QCCheck" DirGen ${figs4RegressDir} "regr-FIGS" DirGen ${nwpAnalysDir} "NWP-ANALYSIS" DirGen ${fwdAnalysDir} "FWD SIMUL on Analyses" DirGen ${orbitMonPath} "QC Monitor" DirGen ${regressRetrDir} "Regression-Based Retrievals" DirGen ${prepSweClimoDir} "SWE Climatology" DirGen ${ffCollocDir} "FF Collocation" DirGen ${imsCollocDir} "IMS Collocation" #-------------------------------------------------------------------------------- # NPP/ATMS special case: to generate NEDT from SDR files #-------------------------------------------------------------------------------- if [[ "${step_rdr2tdrSensor1}" -eq 1 && ${satId} == "npp" && ${rdrType} -eq 1 ]] ; then rdrType2=3 nfile_gatmo=`ls -1 ${rdrSensor1Dir}/GATMO_npp* 2> /dev/null | wc -l` nfile_satms=`ls -1 ${rdrSensor1Dir}/SATMS_npp* 2> /dev/null | wc -l` #---- only do this when SDR files exist and have equal numer of GATMO files if [[ ${nfile_satms} -ge 1 && ${nfile_gatmo} -eq ${nfile_satms} ]] ; then rdr2tdr ${rdrSensor1Dir} ${rdrSensor1List} ${sensor1} ${tdrSensor1Dir} ${sensor1Nedt}\ ${instrumentSensor1File} ${sensor1Wt} ${nOrbits2Process} ${logFile}\ ${rdr2tdrSensor1ControlFile} ${rdr2tdrSensor1Src} ${makeOrNot} ${binPath}\ ${processMode} ${orbitInfo} ${satId} "${rdirExt}" ${controlDataPath}\ ${calibBiasFitFile} ${calibDTRlutFile} ${accessStr} ${formStr} ${rdrType2} #---- clean up ( only need remove TDR to avoid conflict ) rm -f ${tdrSensor1Dir}/TDR* fi fi #-------------------------------------------------------------------------------- # N20/ATMS special case: to generate NEDT from SDR files #-------------------------------------------------------------------------------- if [[ "${step_rdr2tdrSensor1}" -eq 1 && ${satId} == "n20" && ${rdrType} -eq 1 ]] ; then rdrType2=3 nfile_gatmo=`ls -1 ${rdrSensor1Dir}/GATMO_j01* 2> /dev/null | wc -l` nfile_satms=`ls -1 ${rdrSensor1Dir}/SATMS_j01* 2> /dev/null | wc -l` #---- only do this when SDR files exist and have equal numer of GATMO files if [[ ${nfile_satms} -ge 1 && ${nfile_gatmo} -eq ${nfile_satms} ]] ; then rdr2tdr ${rdrSensor1Dir} ${rdrSensor1List} ${sensor1} ${tdrSensor1Dir} ${sensor1Nedt}\ ${instrumentSensor1File} ${sensor1Wt} ${nOrbits2Process} ${logFile}\ ${rdr2tdrSensor1ControlFile} ${rdr2tdrSensor1Src} ${makeOrNot} ${binPath}\ ${processMode} ${orbitInfo} ${satId} "${rdirExt}" ${controlDataPath}\ ${calibBiasFitFile} ${calibDTRlutFile} ${accessStr} ${formStr} ${rdrType2} #---- clean up ( only need remove TDR to avoid conflict ) rm -f ${tdrSensor1Dir}/TDR* fi fi #-------------------------------------------------------------------------------- # N20/ATMS special case: to generate NEDT from SDR files #-------------------------------------------------------------------------------- if [[ "${step_rdr2tdrSensor1}" -eq 1 && ${satId} == "n21" && ${rdrType} -eq 1 ]] ; then rdrType2=3 nfile_gatmo=`ls -1 ${rdrSensor1Dir}/GATMO_j02* 2> /dev/null | wc -l` nfile_satms=`ls -1 ${rdrSensor1Dir}/SATMS_j02* 2> /dev/null | wc -l` #---- only do this when SDR files exist and have equal numer of GATMO files if [[ ${nfile_satms} -ge 1 && ${nfile_gatmo} -eq ${nfile_satms} ]] ; then rdr2tdr ${rdrSensor1Dir} ${rdrSensor1List} ${sensor1} ${tdrSensor1Dir} ${sensor1Nedt}\ ${instrumentSensor1File} ${sensor1Wt} ${nOrbits2Process} ${logFile}\ ${rdr2tdrSensor1ControlFile} ${rdr2tdrSensor1Src} ${makeOrNot} ${binPath}\ ${processMode} ${orbitInfo} ${satId} "${rdirExt}" ${controlDataPath}\ ${calibBiasFitFile} ${calibDTRlutFile} ${accessStr} ${formStr} ${rdrType2} #---- clean up ( only need remove TDR to avoid conflict ) rm -f ${tdrSensor1Dir}/TDR* fi fi #-------------------------------------------------------------------------------- # step: RDR to TDR conversion (sensor1) #-------------------------------------------------------------------------------- if [[ "${step_rdr2tdrSensor1}" -eq 1 ]] ; then rdr2tdr ${rdrSensor1Dir} ${rdrSensor1List} ${sensor1} ${tdrSensor1Dir} ${sensor1Nedt}\ ${instrumentSensor1File} ${sensor1Wt} ${nOrbits2Process} ${logFile}\ ${rdr2tdrSensor1ControlFile} ${rdr2tdrSensor1Src} ${makeOrNot} ${binPath}\ ${processMode} ${orbitInfo} ${satId} "${rdirExt}" ${controlDataPath}\ ${calibBiasFitFile} ${calibDTRlutFile} ${accessStr} ${formStr} ${rdrType} fi #-------------------------------------------------------------------------------- # step: RDR to TDR conversion (sensor2) #-------------------------------------------------------------------------------- if [[ "${step_rdr2tdrSensor2}" -eq 1 ]] ; then rdr2tdr ${rdrSensor2Dir} ${rdrSensor2List} ${sensor2} ${tdrSensor2Dir} ${sensor2Nedt}\ ${instrumentSensor2File} ${sensor2Wt} ${nOrbits2Process} ${logFile}\ ${rdr2tdrSensor2ControlFile} ${rdr2tdrSensor2Src} ${makeOrNot} ${binPath}\ ${processMode} ${orbitInfo} ${satId} "${rdirExt}" ${controlDataPath}\ ${calibBiasFitFile} ${calibDTRlutFile} ${accessStr} ${formStr} ${rdrType} fi #-------------------------------------------------------------------------------- # step: Merge Different Sensor NEDT files into a single file #-------------------------------------------------------------------------------- if [[ "${step_mergeNedt}" -eq 1 ]] ; then mergeNedt ${sensor1Nedt} ${sensor2Nedt} ${nedtBefFMFile} ${logFile}\ ${mergeNedtControlFile} ${mergeNedtSrc} ${makeOrNot} ${binPath} ${satId}\ "${nedtNominalFile}" "${nedtDir}" fi #-------------------------------------------------------------------------------- # step: TDR to SDR conversion (sensor1) #-------------------------------------------------------------------------------- if [[ "${step_tdr2sdrSensor1}" -eq 1 ]] ; then tdr2sdr ${tdrSensor1Dir} ${tdrSensor1List} ${sensor1} ${tdrFormat} ${sdrSensor1Dir}\ ${antennaSensor1File} ${nOrbits2Process} ${logFile} ${tdr2sdrSensor1ControlFile}\ ${makeOrNot} ${binPath} ${processMode} ${orbitInfo} ${tdr2sdrSrc} fi #-------------------------------------------------------------------------------- # step: TDR to SDR conversion (sensor2) #-------------------------------------------------------------------------------- if [[ "${step_tdr2sdrSensor2}" -eq 1 ]] ; then tdr2sdr ${tdrSensor2Dir} ${tdrSensor2List} ${sensor2} ${tdrFormat} ${sdrSensor2Dir}\ ${antennaSensor2File} ${nOrbits2Process} ${logFile} ${tdr2sdrSensor2ControlFile}\ ${makeOrNot} ${binPath} ${processMode} ${orbitInfo} ${tdr2sdrSrc} fi #-------------------------------------------------------------------------------- # step: Footprint Matching (FM) sensor1/sensor2 #-------------------------------------------------------------------------------- if [[ "${step_fm}" -eq 1 ]] ; then fm ${sdrSensor1Dir} ${sdrSensor1List} ${sdrSensor2Dir} ${sdrSensor2List}\ ${sensor1} ${sensor2} ${fmsdrDir} ${outFMAccuracy} ${perfsMonitorPath}\ ${prefixFMAccuracy} ${fmType} ${nScanLineSensor1Skip} ${nScanLineSensor2Skip}\ ${scanLineIndexSensor2TimeColloc} ${nOrbits2Process} ${logFile} ${fmControlFile}\ ${fmSrc} ${makeOrNot} ${binPath} ${processMode} ${orbitInfo} ${satId}\ ${nedtBefFMFile} ${nedtAftFMFile} ${modifyNedtControlFile} ${figs4BiasDir}\ ${identifier} ${outFMAccuracy} ${inputDataPath} ${controlDataPath}\ ${geoLimit} ${minLat} ${maxLat} ${minLon} ${maxLon} fi #-------------------------------------------------------------------------------- # step: Collocate FMSDRs with the IMS Snow/Ice surface type analysis (for nwp step) #-------------------------------------------------------------------------------- if [[ "${step_ims4nwp}" -eq 1 ]] ; then if [[ ${processMode} -eq 0 ]] ; then yyyy=`echo ${orbitInfo} | cut -c1-4` mm=`echo ${orbitInfo} | cut -c5-6` dd=`echo ${orbitInfo} | cut -c7-8` date=${yyyy}-${mm}-${dd} fi yyyy=`echo ${date} | cut -c1-4` let ndays=${ndayLagIms}+1 yyyymmdd=`date -d "${date} $ndays days" +%Y-%m-%d` yyyy=`echo ${yyyymmdd} | cut -c1-4` jjj=`yyyymmdd2jjj $yyyymmdd` #imsFile=${imsAnalysisPath}/"ims"${yyyy}${jjj}"_4km_v1.3.asc" imsFile=${imsAnalysisPath}/"NIC.IMS_v3_"${yyyy}${jjj}"00_4km.asc" echo "imsFile=" ${imsFile} if [ ! -e ${imsFile} ]; then op_msg "Warning in imsColloc: file not found" imsSfcUse=0 fi if [ -e ${imsFile} ]; then imsColloc ${fmsdrDir} "${extResol}" ${fmsdr4imsCollocList} ${imsCollocDir}\ ${imsFile} ${ndayLagIms} ${logFile} ${imsCollocControlFile}\ ${imsCollocSrc} ${makeOrNot} ${binPath} ${processMode} ${orbitInfo}\ ${retrOnOrbitOrSubOrbit} ${fmsdrChoppDir} ${sensorId}\ ${nOrbits2Process} ${version} fi fi #-------------------------------------------------------------------------------- # step: colocation of NWP ECMWF analyses with radiances (for bias) # note: In this step, if you get data allocate problem, you need do this: # ulimit -d unlimited ( on IBM or Linux ) # Or limit datasize unlimited ( on other platforms ) #-------------------------------------------------------------------------------- if [[ "${step_nwp}" -eq 1 ]] ; then checkNWP=`prepNWP ${fmsdrPath} ${rdirAnalysExt} "${extResol}" ${satId} ${ecmwfData}\ ${nwpGenAnalysSrc} ${makeOrNot} ${binPath} ${controlDataPath} ${nwpEcmwfGridPath}\ ${gridSfcNwpAnalysList} ${gridAtmNwpAnalysList}` if [[ ${checkNWP} == "1" ]] ; then echo NWP files needed are complete else echo NWP files needed are NOT complete exit 1 fi nwp ${nwpEcmwfGridPath} ${rdirAnalysExt} ${gridSfcNwpAnalysList} ${gridAtmNwpAnalysList}\ ${fmsdrPath} "${extResol}" ${fmsdr4NwpList} ${nwpAnalysDir} ${topographyFile}\ ${covBkgAtm1File} ${nOrbits2Process} ${logFile} ${nProfs2Fwd} ${grid2nwpControlFile}\ ${nwpGenAnalysSrc} ${makeOrNot} ${binPath} ${sensorId} ${rdirNextAnalysExt}\ ${CRTMcoeffPath} ${ecmwfData}\ ${imsSfcUse} ${imsCollocDir} ${imsColloc4nwpList} ${processMode} ${orbitInfo} fi #-------------------------------------------------------------------------------- # step: Application of forward operator on NWP ECMWF analyses #-------------------------------------------------------------------------------- if [[ "${step_fwd}" -eq 1 ]] ; then fwd ${fwdSrc} ${makeOrNot} ${nwpAnalysDir} ${nwpAnalysList} ${nOrbits2Process}\ ${fwdAnalysDir} ${CRTMcoeffPath} ${instrumentSensor1Sensor2File} ${nProfs2Fwd}\ ${addDeviceNoise} ${nedtAftFMFile} ${monitorFwd} ${logFile} ${fwdControlFile} \ ${binPath} ${satId} ${sensorId} ${ecmwfData} ${fwdCloudOffOrOn} "${rdirExt}" fi #-------------------------------------------------------------------------------- # step: Determination of the bias (by comparing FWD simul and meas) #-------------------------------------------------------------------------------- if [[ "${step_biasGen}" -eq 1 ]] ; then biasGen ${determineBiasSrc} ${nwpAnalysDir} "${extResol}" ${nwpAnalys4BiasList}\ ${fwdAnalysDir} ${fwdAnalys4BiasList} ${fmsdr4BiasDir} ${fmsdr4BiasList}\ ${biasCompuControlFile} ${biasComputeMethod} ${biasFile} ${modelErrFile}\ ${figs4BiasDir} ${nOrbits2Process} ${satId} ${IDL} ${sensorId}\ ${topographyFile} ${binPath} "${modelErrNominalFile}" ${controlDataPath} ${ecmwfData} ${minLat} ${maxLat} ${minLon} ${maxLon} fi #-------------------------------------------------------------------------------- # step: Chopping the FMSDR file into pieces for faster processing #-------------------------------------------------------------------------------- if [[ "${step_choppRadFiles}" -eq 1 ]] ; then chopp ${fmsdrDir} ${fmsdr4ChoppList} ${sensor1}${sensor2} ${fmsdrChoppDir}\ ${nChoppedFilesPerOrbit} ${logFile} ${choppControlFile} ${choppSrc} ${makeOrNot}\ ${binPath} ${processMode} ${orbitInfo} "${extResol}" fi #-------------------------------------------------------------------------------- # step: Collocate FMSDRs with the IMS Snow/Ice surface type analysis (for 1dvar step) #-------------------------------------------------------------------------------- if [[ "${step_ims4retr}" -eq 1 ]] ; then if [[ ${processMode} -eq 0 ]] ; then yyyy=`echo ${orbitInfo} | cut -c1-4` mm=`echo ${orbitInfo} | cut -c5-6` dd=`echo ${orbitInfo} | cut -c7-8` date=${yyyy}-${mm}-${dd} fi yyyy=`echo ${date} | cut -c1-4` let ndays=${ndayLagIms}+1 yyyymmdd=`date -d "${date} $ndays days" +%Y-%m-%d` yyyy=`echo ${yyyymmdd} | cut -c1-4` jjj=`yyyymmdd2jjj $yyyymmdd` #imsFile=${imsAnalysisPath}/"ims"${yyyy}${jjj}"_4km_v1.3.asc" imsFile=${imsAnalysisPath}/"NIC.IMS_v3_"${yyyy}${jjj}"00_4km.asc" echo "imsFile=" ${imsFile} if [ ! -e ${imsFile} ]; then op_msg "Warning in imsColloc: file not found" imsSfcUse=0 fi if [ -e ${imsFile} ]; then imsColloc ${fmsdrDir} "${extResol}" ${fmsdr4imsCollocList} ${imsCollocDir}\ ${imsFile} ${ndayLagIms} ${logFile} ${imsCollocControlFile}\ ${imsCollocSrc} ${makeOrNot} ${binPath} ${processMode} ${orbitInfo}\ ${retrOnOrbitOrSubOrbit} ${fmsdrChoppDir} ${sensorId}\ ${nOrbits2Process} ${version} fi fi #-------------------------------------------------------------------------------- # step: Apply regression-based algorithms on FM-SDR radiances #-------------------------------------------------------------------------------- if [[ "${step_externalDataFromRegress}" -eq 1 ]] ; then applyRegress ${fmsdrDir} "${extResol}" ${fmsdr4ApplyRegressList} ${regressRetrDir}\ ${topographyFile} ${covBkgAtm1File} ${covBkgSfc1File} ${logFile} ${regressControlFile}\ ${applyRegressAlgSrc} ${makeOrNot} ${binPath} ${processMode} ${orbitInfo}\ ${regressCoeffOceanClwFile} ${regressCoeffSeaIceClwFile} ${regressCoeffLandClwFile} ${regressCoeffSnowClwFile}\ ${regressCoeffOceanTskinFile} ${regressCoeffSeaIceTskinFile} ${regressCoeffLandTskinFile} ${regressCoeffSnowTskinFile}\ ${regressCoeffOceanTpwFile} ${regressCoeffSeaIceTpwFile} ${regressCoeffLandTpwFile} ${regressCoeffSnowTpwFile}\ ${regressCoeffOceanEmFile} ${regressCoeffSeaIceEmFile} ${regressCoeffLandEmFile} ${regressCoeffSnowEmFile}\ ${regressCoeffOceanWvFile} ${regressCoeffSeaIceWvFile} ${regressCoeffLandWvFile} ${regressCoeffSnowWvFile}\ ${regressCoeffOceanTempFile} ${regressCoeffSeaIceTempFile} ${regressCoeffLandTempFile} ${regressCoeffSnowTempFile}\ ${regressCoeffOceanGwpFile} ${regressCoeffSeaIceGwpFile} ${regressCoeffLandGwpFile} ${regressCoeffSnowGwpFile}\ ${retrOnOrbitOrSubOrbit} ${fmsdrChoppDir} ${sensorId}\ ${nOrbits2Process} ${biasFileToUse} ${tune1File} ${version} fi #-------------------------------------------------------------------------------- # step: EDRs retrieval from the FM-SDRs #-------------------------------------------------------------------------------- if [[ "${step_fmsdr2edr}" -eq 1 ]] ; then fmsdr2edr ${fmsdr2edrSrc} ${makeOrNot} ${retrOnWhichSDR} ${fmsdrDir}\ ${fwdAnalysDir} ${retrOnOrbitOrSubOrbit} ${fmsdrChoppDir} ${fmsdrList}\ ${nOrbits2Process} ${externalDataAvailable} ${externalDataSrc} ${nwpAnalysDir}\ ${nwpAnalysRetrList} ${regressRetrDir} ${edrDir} ${nProfs2Retr}\ ${monitorIterative} ${nAttempts} ${monitorRetrieval} ${geoLimit}\ ${minLat} ${maxLat} ${minLon} ${maxLon} ${cend} ${sensorId}\ ${tune1File} ${tune2File} ${covBkgAtm1File} ${covBkgAtm2File}\ ${covBkgSfc1File} ${covBkgSfc2File} ${modelErrFile1ToUse} ${modelErrFile2ToUse} \ ${extBkgAtmFile} ${extBkgAtmUse}\ ${imsSfcUse} ${imsCollocDir} ${imsColloc4retrList}\ ${nedtAftFMFile} ${monitorFile} ${topographyFile}\ ${CRTMcoeffPath} ${logFile} ${fmsdr2edrControlFile} ${useCPU} ${processMode}\ ${orbitInfo} ${binPath} ${biasFileToUse} ${version} "${extResol}" "${rdirExt}" \ ${TBbias_NNModelFile} ${TPW_NNModelFile} ${Tskin_NNModelFile} fi #-------------------------------------------------------------------------------- # step: Merge the mini EDR files into full orbit file #-------------------------------------------------------------------------------- if [[ "${step_mergeEdr}" -eq 1 ]] ; then mergeEdr ${edrDir} "${extResol}" ${edr4MergeList} ${sensor1}${sensor2}\ ${nChoppedFilesPerOrbit} ${logFile} ${mergeEdrControlFile} ${mergeEdrSrc}\ ${makeOrNot} ${binPath} ${processMode} ${orbitInfo} fi #-------------------------------------------------------------------------------- # step: Prepare files with SWE climatology information (for vipp) #-------------------------------------------------------------------------------- if [[ "${step_prepSweClimo}" -eq 1 ]] ; then prepSweClimo ${fmsdrDir} "${extResol}" ${fmsdr4PrepSweList} ${prepSweClimoDir}\ ${SweClimoMeanFile} ${SweClimoStdevFile} ${logFile} ${prepSweClimoControlFile}\ ${prepSweSrc} ${makeOrNot} ${binPath} ${processMode} ${orbitInfo}\ ${retrOnOrbitOrSubOrbit} ${fmsdrChoppDir} ${sensorId}\ ${nOrbits2Process} ${version} fi #-------------------------------------------------------------------------------- # step: Collocate FMSDRs with the VIIRS-based forest fraction (for vipp) #-------------------------------------------------------------------------------- if [[ "${step_ffColloc}" -eq 1 ]] ; then ffColloc ${fmsdrDir} "${extResol}" ${fmsdr4ffCollocList} ${ffCollocDir}\ ${ffFile} ${ffLatLonFile} ${logFile} ${ffCollocControlFile}\ ${ffCollocSrc} ${makeOrNot} ${binPath} ${processMode} ${orbitInfo}\ ${retrOnOrbitOrSubOrbit} ${fmsdrChoppDir} ${sensorId}\ ${nOrbits2Process} ${version} fi #-------------------------------------------------------------------------------- # step: To generate vertical integrated products #-------------------------------------------------------------------------------- if [[ "${step_vipp}" -eq 1 ]] ; then vipp ${edrList} ${edrDir} ${prepSweClimoDir} ${prepSweClimo4VippList}\ ${sweClimoUse} ${ffCollocDir} ${ffColloc4VippList} ${ffUse}\ ${depDir} ${logFile} ${nOrbits2Process} ${nProfs2Retr}\ ${sensorId} ${processMode} ${orbitInfo} "${extResol}" ${vippControlFile}\ ${vippSrc} ${binPath} ${siceEmissCatalogFile} ${snowEmissCatalogFile} fi #-------------------------------------------------------------------------------- # step: To generate sfr products #-------------------------------------------------------------------------------- if [[ "${step_sfr}" -eq 1 ]] ; then sfr ${satId} ${sensor1} ${fmsdrDir} "${extResol}" ${fmsdr4SfrList} ${sfrDir} ${sfrDataPath}\ ${logFile} ${sfrSrc} ${makeOrNot} ${binPath}\ ${processMode} ${orbitInfo} ${retrOnOrbitOrSubOrbit} ${fmsdrChoppDir}\ ${nOrbits2Process} ${sfrControlFile} ${nwpSfrGridPath} ${gridNwpSfrList}\ ${nProfs2Retr} ${sfr2depSrc} ${sfr2depControlFile} ${sfrList} ${depDir} ${depList} fi #-------------------------------------------------------------------------------- # step: To generate gridded products #-------------------------------------------------------------------------------- if [[ "${step_grid}" -eq 1 ]] ; then gridGen_parallel ${processMode} ${edrDir} ${depDir} ${orbitInfo} ${edrList} ${depList}\ "${extResol}" ${satId} ${fileExt} ${gridSrc} ${gridDir} ${makeOrNot} ${binPath}\ ${gridControlFile} ${gridFactor} ${minLat} ${maxLat} ${minLon} ${maxLon}\ ${fmType} "${edrGridStr}" "${edrP2PStr}" "${depGridStr}" "${depP2PStr}" fi #-------------------------------------------------------------------------------- # step: To convert EDR & DEP into netcdf4 format #-------------------------------------------------------------------------------- if [[ "${step_nc}" -eq 1 ]] ; then mirs2nc_parallel ${edrDir} ${depDir} ${ncDir} ${binPath} ${satId} ${rdrSensor1List} ${rdrType} ${HDF5BIN} ${production_site} ${production_environment} fi #-------------------------------------------------------------------------------- # step: EDRs Figures Generation #-------------------------------------------------------------------------------- if [[ "${step_figsGen}" -eq 1 ]] ; then figsGen_parallel ${satId} ${gridFactor} ${gridDir} ${gridSrc} ${figsDir} ${IDL}\ ${identifier} ${figsGenControlFile} ${processMode} ${controlDataPath} ${version}\ ${minLat} ${maxLat} ${minLon} ${maxLon} fi #-------------------------------------------------------------------------------- # step: Bias Figures Generation (NWP ECMWF) #-------------------------------------------------------------------------------- if [[ "${step_biasFigsGen}" -eq 1 ]] ; then biasFigsGen_parallel ${edr4BiasDir} ${nwpAnalysDir} ${fwdAnalysDir} ${edr4BiasList}\ ${nwpAnalys4BiasList} ${fwdAnalys4BiasList} ${processMode} ${orbitInfo}\ "${extResol}" ${satId} ${rdirAnalysExt} ${gridSrc} ${grid4BiasDir} ${makeOrNot}\ ${binPath} ${nwpGridControlFile} ${fwdGridControlFile} ${biasGridControlFile}\ ${gridFactor} ${gridSrc} ${figs4BiasDir} ${IDL} ${rdirAnalysExt}\ ${figsGenControlFile} ${controlDataPath} ${dep4BiasDir} ${dep4BiasList}\ ${version} ${biasPath} ${minLat} ${maxLat} ${minLon} ${maxLon} ${ecmwfData}\ ${fmType} "${nwpGridStr}" "${nwpP2PStr}" "${nedtExt}" fi #-------------------------------------------------------------------------------- # step: Data monitoring (plots) #-------------------------------------------------------------------------------- if [[ "${step_dataMonitor}" -eq 1 ]] ; then qcRetrieval ${depDir} ${depList} ${orbitMonPath} ${IDL} ${gridSrc}\ ${controlDataPath}/qcRetrieval_abnormal_${satId}_${fileExt}\ ${controlDataPath}/qcRetrieval_namelist_${satId}_${fileExt}\ "${email}" "${website}" ${controlDataPath} ${satId} ${date} ${binPath} dataQualityMonitor ${nedtSensor1Sensor2Path} ${nedtList} ${nedtMonitorSrc} ${IDL}\ ${orbitMonPath} ${figsDir} ${processMode} ${fileExt} ${satId} ${controlDataPath} ${perfsMonitorPath} fi #-------------------------------------------------------------------------------- # step: Cleaning Up #-------------------------------------------------------------------------------- if [[ "${step_clean}" -eq 1 ]] ; then if [[ "${satId}" == "f16" || "${satId}" == "f17" || "${satId}" == "f18" || "${satId}" == "f19" || "${satId}" == "gpm" || "${satId}" == "npp" || "${satId}" == "n20" || ${satId} == "n21" || ${satId} == "metopSGA1" ]] ; then tdrSensor2Path='Dummy' sdrSensor2Path='Dummy' rdr2tdrSensor2Src='Dummy' nedtSensor2Path='Dummy' nedtSensor1Sensor2Path='Dummy' fi clean ${satId} ${maxDaysArchived} ${tdrSensor1Path} ${tdrSensor2Path} ${sdrSensor1Path}\ ${sdrSensor2Path} ${fmsdrPath} ${choppPath} ${edrPath} ${depPath}\ ${figsPath} ${nwpAnalysPath} ${fwdAnalysPath} ${regressRetrPath}\ ${perfsMonitorPath} ${makeOrNot} ${rdr2tdrSensor1Src} ${rdr2tdrSensor2Src}\ ${mergeNedtSrc} ${tdr2sdrSrc} ${fmSrc} ${nwpGenAnalysSrc} ${fwdSrc} \ ${applyRegressAlgSrc} ${fmsdr2edrSrc} ${controlDataPath} ${inputDataPath} ${binPath}\ ${logPath} ${prefixFMAccuracy} ${determineBiasSrc} \ ${applyRegressAlgSrc} ${rootPath} ${processMode} ${makeClean} ${nedtSensor1Path}\ ${nedtSensor2Path} ${nedtSensor1Sensor2Path} ${regressPath} ${biasPath} ${gridPath} fi