from cris_data_finder import * import datetime import re from static_info import * def any_df(start,end,sat): if sat in tr_ver_map: return trop_df(start,end,sat) elif sat in ['J02','J01','NPP']: return atdf(start,end,sat) elif 'AWS' in sat: return df_scdr.df_scdr( start, end, dtype='AWS-MWR-L1B', reduce_type='AWS1-MWR-1B', timefinder=aws_startend, key_length=23, sat='AWS1', skey_func=aws_reduce, ) elif sat in ['TS02','TS03','TS04','TS05','TS06','TS07']: return cdp_df(start,end,sat) elif 'WSF' in sat: return wsf_df(start,end,sat) elif 'MET' in sat: return df_scdr.df_scdr( start, end, dtype='METOP-SGA1-MWS-L1B', reduce_type='SGA1-MWS-1B', timefinder=aws_startend, key_length=23, sat=sat, usesat=False, # sat='METOP-SGA1', skey_func=aws_reduce, ) # W_NO-KSAT-Tromso,SAT,AWS1-MWR-1B-RAD_C_OHB__20250825122909_G_O_20250825103754_20250825121428_C_N____.nc aws_re = re.compile(r'([A-Z0-9-]{15})_(\\S{6})_(\\d{14})_(\\S{3})_(\\d{14})_(\\d{14})_(\\S{7})\\.nc') def aws_startend(fn): reg = aws_re.findall(fn) pair = [None,None] if reg is not None and len(reg) > 0: reg = reg[0] pair = [datetime.datetime(int(reg[4][0:4]), int(reg[4][4:6]), int(reg[4][6:8]), int(reg[4][8:10]), int(reg[4][10:12]), int(reg[4][12:14]), ), datetime.datetime(int(reg[5][0:4]), int(reg[5][4:6]), int(reg[5][6:8]), int(reg[5][8:10]), int(reg[5][10:12]), int(reg[5][12:14]), ), ] return pair def aws_reduce(fn,name,kl): reg = aws_re.findall(fn) if reg is not None and len(reg) > 0: reg = reg[0] return reg[0]+reg[4]+reg[5] return fn mtgi_re = re.compile('([A-Z0-9-]{15})_(\\S{6})_(\\d{14})_(\\S{3,10})_(\\d{14})_(\\d{14})_(\\S*)\\.nc') def mtgi_startend(fn): reg = mtgi_re.findall(fn) pair = [None,None] if reg is not None and len(reg) > 0: reg = reg[0] pair = [datetime.datetime(int(reg[4][0:4]), int(reg[4][4:6]), int(reg[4][6:8]), int(reg[4][8:10]), int(reg[4][10:12]), int(reg[4][12:14]), ), datetime.datetime(int(reg[5][0:4]), int(reg[5][4:6]), int(reg[5][6:8]), int(reg[5][8:10]), int(reg[5][10:12]), int(reg[5][12:14]), ), ] return pair def mtg_reduce(fn,name,kl): reg = mtgi_ew.findall(fn) if reg is not None and len(reg) > 0: reg = reg[0] return reg[0]+reg[4]+reg[5] return fn def mtgi_df(start,end,path='/data/smcd10/CrIS/pbeierle/MTG-IRS/IRS-L1B/'): return df_multi.df_multi( start, end, sat='MTG', timefinder=mtgi_startend, scanorder=[df_scdr.df_scdr,df_cache.df_cache], extras = [ {'sat': 'MTG','dtype': f'MTG-IRS-L1B','reduce_type':f'+IRS-1B-PC','key_length':130,'usesat':False}, {'base':path,'path':path,'verify_re':mtgi_re}, ] ) # scantime = pull2.data['time'][:,40] return df_cache.df_cache( start, end, base=path, path=path, timefinder=mtgi_startend, verify_re=mtgi_re, ) cdp_re = re.compile(r'([A-Z0-9-]{5,6})\.(\S+)\.(V\S+)\.(\w{3})\.ST(\d{8})-(\d{6})\.ET(\d{8})-(\d{6})\.CT(\d{8})-(\d{6})\.nc') def cdp_startend(fn): reg = cdp_re.findall(fn) pair = [None,None] if reg is not None and len(reg) > 0: reg = reg[0] pair = [datetime.datetime(int(reg[4][0:4]), int(reg[4][4:6]), int(reg[4][6:8]), int(reg[5][0:2]), int(reg[5][2:4]), int(reg[5][4:6]), ), datetime.datetime(int(reg[6][0:4]), int(reg[6][4:6]), int(reg[6][6:8]), int(reg[7][0:2]), int(reg[7][2:4]), int(reg[7][4:6]), ), ] return pair wsfm_re = re.compile(r'([A-Z0-9\-]{4,11})_(\d+)_d(\d{8})_t(\d{6})_e(\d{6})') def get_wsfm_startend(infile): reg = wsfm_re.findall(infile)[0] pair = [datetime.datetime(int(reg[2][0:4]), int(reg[2][4:6]), int(reg[2][6:8]), int(reg[3][0:2]), int(reg[3][2:4]), int(reg[3][4:6])), datetime.datetime(int(reg[2][0:4]), int(reg[2][4:6]), int(reg[2][6:8]), int(reg[4][0:2]), int(reg[4][2:4]), int(reg[4][4:6])), ] if pair[0] > pair[1]: pair[1] += datetime.timedelta(days=1) return pair def wsf_df(start,end,sat,dtype='WSFM01_NAT_SDR'): return df_scdr.df_scdr( start, end, sat=sat, dtype=dtype, usesat=False, reduce_type='WSFM_01', timefinder=get_wsfm_startend, key_length=34, ) def cdp_df(start,end,sat,dtype='L1B-TB',version_string='.V01'): mangle_dtype = dtype[dtype.index('-')+1:]+'_'+dtype[:dtype.index('-')] mangle_sat = sat[0]+'M'+sat[1:] return df_multi.df_multi( start, end, sat=sat, timefinder=cdp_startend, dtype=f'{sat}_{mangle_dtype}', scanorder=[df_scdr.df_scdr,df_tropics_cache.df_tropics_cache], extras = [ { 'reduce_type': f'{mangle_sat}.{dtype[1:]}{version_string}', 'key_length':58, 'usesat':False, },{ 'regex':cdp_re, 'base':'/data/www/data/cdp_pilot/TMS/', 'path':'{self.base}/{self.dtype}/{self._ymdc()}/', 'dtype':sat+'_'+dtype, 'reduce_type': f'{mangle_sat}.{dtype[1:]}{version_string}', } ] ) def trop_df(start,end,sat): ver = tr_ver_map[sat] return df_multi.df_multi( start, end, sat=sat, timefinder=df_tropics_cache.tr_startend, scanorder=[df_tropics_cache.df_tropics_cache,df_tropics_cache.df_tropics_cache,df_scdr.df_scdr], extras = [ {'ver': ver,'sat': sat}, {'ver': ver,'sat': sat,'base':'/data/smcd10/TROPICS/cache','path':'{self.base}/{self.sat}/TROPICS_L1B/{self._ymds()}/',}, {'sat': sat,'dtype': f'{sat}_BRTT_L1B','reduce_type':f'{sat}.BRTT.L1B','key_length':72,'usesat':False}, ] ) epoch = datetime.datetime.fromisoformat('2000-01-01') # scantime = pull2.data['time'][:,40] # dtscantime = np.array([epoch + datetime.timedelta(seconds=x) for x in scantime]) def crdf(start,end,sat,cache='/data/smcd10/CrIS/wayne.mccullough/cache'): return df_multi.df_multi(start, end, dtype='SCRIF', geotype='GCRSO', sat=sat, scanorder=[df_scdr.df_scdr,df_cache.df_cache,df_boto.df_boto], extras = [ {}, {'base':cache, 'path':'{self.base}/{self.sat}/{self.dtype}/{self._ymds()}/'}, {'base':cache}, ] ) def atdf(start,end,sat,cache='/data/smcd10/CrIS/wayne.mccullough/cache'): return df_multi.df_multi(start, end, dtype='SATMS', geotype='GATMO', sat=sat, scanorder=[df_cache.df_cache,df_scdr.df_scdr,df_cache.df_cache,df_boto.df_boto], extras = [{'base': '/data/data263/OPS_DATA' }, {}, {'base':cache, 'path':'{self.base}/{self.sat}/{self.dtype}/{self._ymds()}/'}, {'base':cache, 'path':'{self.base}/{self.sat}/{self.dtype}/{self._ymds()}/'}, ] ) def tatdf(start,end,sat,cache='/data/smcd10/CrIS/wayne.mccullough/cache'): return df_multi.df_multi(start, end, dtype='TATMS', geotype='GATMO', sat=sat, scanorder=[df_cache.df_cache,df_scdr.df_scdr,df_cache.df_cache,df_boto.df_boto], extras = [{'base': '/data/data263/OPS_DATA' }, {}, {'base':cache, 'path':'{self.base}/{self.sat}/{self.dtype}/{self._ymds()}/'}, {'base':cache, 'path':'{self.base}/{self.sat}/{self.dtype}/{self._ymds()}/'}, ] ) def gfs_df(start,end, mycache='/data/smcd10/CrIS/wayne.mccullough/cache'): mypath='{self.base}/gfs.0p25/{self._year()}/{self._ymd()}/' return df_multi.df_multi( start, end, dtype='GFS_0P25', reduce_type='gfs.t', reduce=df_gfs.reduce, geotype=None, sat=None, timefinder=df_gfs.gfs_both_sten, scanorder=[df_gfs.df_scdr_gfs,df_cache.df_cache,df_gfs.df_boto_gfs], extras = [ {'timefinder':df_gfs.gfs_startend}, {'base':mycache, 'path':mypath,'timefinder':df_gfs.gfs_cache_startend,'verify_re':df_gfs.gfs_short_re}, {'base':mycache, 'path':mypath}, ], ) ''' def get_ch_str(sat): if sat in tr_ver_map: return trop_ch_str elif sat in ['J02','J01','NPP']: return atms_ch_str elif 'AWS' in sat: return awspfm_ch_str elif 'TS' in sat or 'TMS' in sat: return tms_ch_str elif 'METOP-SGA1' in sat: return metsg_mws_ch_str def get_inst_name(sat): if sat in tr_ver_map: return 'TROPICS' elif sat in ['J02','J01','NPP']: return 'ATMS' elif 'AWS' in sat: return 'AWSMWS' elif 'TS' in sat or 'TMS' in sat: return 'TMS' elif 'METOP-SGA1' in sat: return 'MWS' ''' # scantime = pull.data['time'][:,47] # dtscantime = np.array([ietepoch + datetime.timedelta(seconds=x/1000000) for x in scantime])