/* Name- processCRIMSS.c Language- C Type- MAIN Version- 1.0 Date- 4/08/2018 Programmer- Mike Pettey (IMSG) Function- This program extracts data from P72 EDR files in HDF5 format and writes the data to an EDGE orbital file. The selected data is defined in the run script. Also in the run script are the starting date (YYYYMMDD) and hour and the ending date and hour. */ #include #include #include #include #include #include #include "nprovs_to_netcdf.h" #define ERR(e) {printf("Error: %s, error num = %d\n", nc_strerror(e),e);} #define NDIMS 1 int defineVariable(int nc_group_id, char *var_name, nc_type var_type, int ndims, int *dimids, char *attr_string, char *attr); void writeVariableFloat(int group_id, int var_id, size_t *index, size_t *num_vals, float value); void writeVariableShort(int group_id, int var_id, size_t *index, size_t *num_vals, short value); void writeVariableInteger(int group_id, int var_id, size_t *index, size_t *num_vals, long value); void writeArrayFloat(int group_id, int var_id, size_t *index, size_t *num_vals, float *value); void writeVariableByte(int group_id, int var_id, size_t *index, size_t *num_vals, char value); void writeAttributeShort(int grp, char *attrVariable, short attrValue); void writeAttributeText(int grp, char *attrVariable, char *attrValue); void processCRIMSS(int date_to_process, struct file_data *files, struct platform *platform_ptr, int num_collocations, int platform_index, int data_type, int nc_date_group) { int n, i, date_type, recnum, collocation_num, retval; int nc_group_id, collocation_group_id; size_t num_colls, index[1], num_vals[1], index_2D[2], num_vals_2D[2]; float missing_float, latitude, longitude, data101[101], data42[42], data22[22]; int year, month, day, hour, minute, second; long offset, yyyymmdd, hhmmss; char col_dir_name[50]; char *string[1]; short *buffer; int col_scalar_dim, scalar_dim, dim_levels, dim_templevels, dim_wvaplevels; int dimid_scalar[1], col_dimid_scalar[1], dimid_levels[2], dimid_templevels[2], dimid_wvaplevels[2]; int col_vid_lat, col_vid_lon, col_vid_date, col_vid_time; int vid_lat, vid_lon, vid_date, vid_time; int vid_node, vid_dataframe, vid_scanline, vid_fov, vid_orbit; int vid_solzen, vid_solazimuth, vid_satzen, vid_satazimuth; int vid_surf_height, vid_land_frac, vid_surf_press, vid_skin_temp; int vid_noise, vid_profdiff, vid_iterations; int vid_qc, vid_cloud, vid_ice, vid_rain, vid_cell, vid_daynight; int vid_irmw_converge, vid_mw_converge, vid_lte, vid_terrain; int vid_irmw_chi, vid_mw1_chi, vid_mw2_chi; int vid_press, vid_temppress, vid_wvappress; int vid_temp, vid_wvap, vid_ir_temp, vid_mw_temp, vid_ir_wvap, vid_mw_wvap; FILE *in; num_colls = (size_t)num_collocations; missing_float = -32768.0; // If this is the baseline platform, then create another subgroup // for the collocation information if (platform_index == 0) { sprintf(col_dir_name, "Collocation_Info"); if ((retval = nc_def_grp(nc_date_group, col_dir_name, &collocation_group_id))) { if (retval != -42) ERR(retval); } writeAttributeShort(collocation_group_id, "Baseline_Data_Type", platform_ptr->type); writeAttributeText(collocation_group_id, "Baseline_Platform", platform_ptr->description); writeAttributeShort(collocation_group_id, "Baseline_Platform_ID", platform_ptr->id); } // Create a subgroup for this system platform if ((retval = nc_def_grp(nc_date_group, platform_ptr->dir_name, &nc_group_id))) { ERR(retval); } // Set the platform type attribute to the data group type writeAttributeText(nc_group_id, "Platform_Name", platform_ptr->description); // clean_description? writeAttributeShort(nc_group_id, "Platform_ID", platform_ptr->id); writeAttributeShort(nc_group_id, "Platform_Type", platform_ptr->type); writeAttributeText(nc_group_id, "Platform_Data_Source", "CRIMSS"); //writeAttributeText(nc_group_id, "Platform_NPROVS_Source_Name", nprovs_name); writeAttributeText(nc_group_id, "Platform_NPROVS_Source_Name", platform_ptr->description); // clean_description? // Define the dimensions if ((retval = nc_def_dim(nc_group_id, "Num_Collocations", num_colls, &scalar_dim))) ERR(retval); dimid_scalar[0] = scalar_dim; if ((retval = nc_def_dim(nc_group_id, "Levels", 101, &dim_levels))) ERR(retval); dimid_levels[0] = scalar_dim; dimid_levels[1] = dim_levels; if ((retval = nc_def_dim(nc_group_id, "Temperature_Levels", 42, &dim_templevels))) ERR(retval); dimid_templevels[0] = scalar_dim; dimid_templevels[1] = dim_templevels; if ((retval = nc_def_dim(nc_group_id, "Moisture_Levels", 22, &dim_wvaplevels))) ERR(retval); dimid_wvaplevels[0] = scalar_dim; dimid_wvaplevels[1] = dim_wvaplevels; if (platform_index == 0) { if ((retval = nc_def_dim(collocation_group_id, "Num_Collocations", num_colls, &col_scalar_dim))) ERR(retval); col_dimid_scalar[0] = col_scalar_dim; } // Define the variables vid_lat = defineVariable(nc_group_id, "latitude", NC_FLOAT, 1, dimid_scalar, "units", "degrees_north"); vid_lon = defineVariable(nc_group_id, "longitude", NC_FLOAT, 1, dimid_scalar, "units", "degrees_east"); vid_date = defineVariable(nc_group_id, "date", NC_INT, 1, dimid_scalar, "format", "yyyymmdd"); vid_time = defineVariable(nc_group_id, "time", NC_INT, 1, dimid_scalar, "format", "hhmmss"); vid_node = defineVariable(nc_group_id, "orbital_node", NC_BYTE, 1, dimid_scalar, "values", "0=ascending, 1=descending"); vid_dataframe = defineVariable(nc_group_id, "data_frame", NC_SHORT, 1, dimid_scalar, NULL, NULL); vid_scanline = defineVariable(nc_group_id, "scan_line", NC_SHORT, 1, dimid_scalar, NULL, NULL); vid_fov = defineVariable(nc_group_id, "field_of_view", NC_SHORT, 1, dimid_scalar, NULL, NULL); vid_orbit = defineVariable(nc_group_id, "orbit_number", NC_INT, 1, dimid_scalar, NULL, NULL); vid_solazimuth = defineVariable(nc_group_id, "solar_azimuth_angle", NC_FLOAT, 1, dimid_scalar, "units", "degrees"); vid_solzen = defineVariable(nc_group_id, "solar_zenith_angle", NC_FLOAT, 1, dimid_scalar, "units", "degrees"); vid_satazimuth = defineVariable(nc_group_id, "satellite_azimuth_angle", NC_FLOAT, 1, dimid_scalar, "units", "degrees"); vid_satzen = defineVariable(nc_group_id, "satellite_zenith_angle", NC_FLOAT, 1, dimid_scalar, "units", "degrees"); vid_surf_height = defineVariable(nc_group_id, "surface_height", NC_FLOAT, 1, dimid_scalar, NULL, NULL); vid_land_frac = defineVariable(nc_group_id, "land_fraction", NC_FLOAT, 1, dimid_scalar, NULL, NULL); vid_surf_press = defineVariable(nc_group_id, "surface_pressure", NC_FLOAT, 1, dimid_scalar, NULL, NULL); vid_skin_temp = defineVariable(nc_group_id, "skin_temperature", NC_FLOAT, 1, dimid_scalar, NULL, NULL); vid_noise = defineVariable(nc_group_id, "ir_noise_amplification", NC_FLOAT, 1, dimid_scalar, NULL, NULL); vid_profdiff = defineVariable(nc_group_id, "profile_difference", NC_FLOAT, 1, dimid_scalar, NULL, NULL); vid_iterations = defineVariable(nc_group_id, "iterations", NC_FLOAT, 1, dimid_scalar, NULL, NULL); vid_qc = defineVariable(nc_group_id, "overall_quality_flag", NC_BYTE, 1, dimid_scalar, "values", "0=high (ir+mw), 1=low (ir only), 2=low (mw only), 3=poor (non converged"); vid_cloud = defineVariable(nc_group_id, "cloudiness", NC_BYTE, 1, dimid_scalar, "values", "0=clear, 1=partly cloudy, 2=cloudy"); vid_ice = defineVariable(nc_group_id, "ice_indicator", NC_BYTE, 1, dimid_scalar, "values", "0=no ice, 1=ice on water"); vid_rain = defineVariable(nc_group_id, "rain_flag", NC_BYTE, 1, dimid_scalar, "values", "0=no precipiation, 1=precipiation detected"); vid_cell = defineVariable(nc_group_id, "retrieval_cell_size", NC_BYTE, 1, dimid_scalar, "values", "0 = 9 fovs used, 1 = 4 fovs used, 2 = 1 fov used, 3 = no retrieval"); vid_daynight = defineVariable(nc_group_id, "day_night_flag", NC_BYTE, 1, dimid_scalar, "values", "0=night, 1=day"); vid_irmw_converge = defineVariable(nc_group_id, "irmw_convergence", NC_BYTE, 1, dimid_scalar, "values", "0=converged, 1=did not converge"); vid_mw_converge = defineVariable(nc_group_id, "mw_convergence", NC_BYTE, 1, dimid_scalar, "values", "0=converged, 1=did not converge"); vid_lte = defineVariable(nc_group_id, "non_lte_flag", NC_BYTE, 1, dimid_scalar, "values", "0=lte, 1=non-lte"); vid_terrain = defineVariable(nc_group_id, "surface_type", NC_BYTE, 1, dimid_scalar, "values", "0=sea, 1=land, 2=coast"); vid_irmw_chi = defineVariable(nc_group_id, "irmw_chi_square", NC_FLOAT, 1, dimid_scalar, NULL, NULL); vid_mw1_chi = defineVariable(nc_group_id, "mw_1_chi_square", NC_FLOAT, 1, dimid_scalar, NULL, NULL); vid_mw2_chi = defineVariable(nc_group_id, "mw_2_chi_square", NC_FLOAT, 1, dimid_scalar, NULL, NULL); vid_temppress = defineVariable(nc_group_id, "temperature_pressure", NC_FLOAT, 2, dimid_templevels, "units", "hPa"); vid_temp = defineVariable(nc_group_id, "temperature", NC_FLOAT, 2, dimid_templevels, "units", "K"); vid_wvappress = defineVariable(nc_group_id, "moisture_pressure", NC_FLOAT, 2, dimid_wvaplevels, "units", "hPa"); vid_wvap = defineVariable(nc_group_id, "water_vapor_mixing_ratio", NC_FLOAT, 2, dimid_wvaplevels, "units", "g/kg"); vid_press = defineVariable(nc_group_id, "pressure", NC_FLOAT, 2, dimid_levels, "units", "hPa"); vid_ir_temp = defineVariable(nc_group_id, "infrared_temperature", NC_FLOAT, 2, dimid_levels, "units", "K"); vid_mw_temp = defineVariable(nc_group_id, "microwave_temperature", NC_FLOAT, 2, dimid_levels, "units", "K"); vid_ir_wvap = defineVariable(nc_group_id, "infrared_water_vapor_mixing_ratio", NC_FLOAT, 2, dimid_levels, "units", "g/kg"); vid_mw_wvap = defineVariable(nc_group_id, "microwave_water_vapor_mixing_ratio", NC_FLOAT, 2, dimid_levels, "units", "g/kg"); if (platform_index == 0) { col_vid_lat = defineVariable(collocation_group_id, "latitude", NC_FLOAT, 1, col_dimid_scalar, "units", "degrees_north"); col_vid_lon = defineVariable(collocation_group_id, "longitude", NC_FLOAT, 1, col_dimid_scalar, "units", "degrees_east"); col_vid_date = defineVariable(collocation_group_id, "date", NC_INT, 1, col_dimid_scalar, "format", "yyyymmdd"); col_vid_time = defineVariable(collocation_group_id, "time", NC_INT, 1, col_dimid_scalar, "format", "hhmmss"); } // Allocate the memory used by the data buffer buffer = (short *)malloc(platform_ptr->length); collocation_num = -1; // Loop through each record in all of the files and read the data for // this data group into the buffer for (date_type=0; date_type<3; date_type++) { if (date_type == 0) { if ((in=fopen("prev_day.file", "r")) == NULL) { printf("\n\nThe previous day file could not be opened for input.\n"); printf("It will be skipped.\n\n"); in = NULL; } } else if (date_type == 1) { if ((in=fopen("in.file", "r")) == NULL) { printf("\n\nThe input file could not be opened for input.\n"); printf("It will be skipped.\n\n"); in = NULL; } } else { if ((in=fopen("next_day.file", "r")) == NULL) { printf("\n\nThe next day file could not be opened for input.\n"); printf("It will be skipped.\n\n"); in = NULL; } } for (recnum=0; recnumoffset; fseek(in, offset, SEEK_SET); fread(buffer, platform_ptr->length, 1, in); // Process this record if it matches the date to process if (files[date_type].use_record[recnum] == TRUE) { collocation_num++; // Index and num_vals are used to save a single value within the // variable array index[0] = collocation_num; num_vals[0] = 1; // Latitude and longitude if (buffer[3] != -32768) latitude = buffer[3] / 128.0; else latitude = -32768.0; if (buffer[4] != -32768) longitude = buffer[4] / 128.0; else longitude = -32768.0; writeVariableFloat(nc_group_id, vid_lat, index, num_vals, latitude); writeVariableFloat(nc_group_id, vid_lon, index, num_vals, longitude); // Date and time if ((buffer[5] != -32768) && (buffer[6] != -32768) && (buffer[7] != -32768) && (buffer[8] != -32768)) { year = buffer[5]; month = buffer[6] / 100; day = buffer[6] % 100; hour = buffer[7]; minute = buffer[8] / 100; second = buffer[8] % 100; yyyymmdd = (year * 10000) + (month * 100) + day; hhmmss = (hour * 10000) + (minute * 100) + second; } else { yyyymmdd = -32768; hhmmss = -32768; } writeVariableInteger(nc_group_id, vid_date, index, num_vals, yyyymmdd); writeVariableInteger(nc_group_id, vid_time, index, num_vals, hhmmss); // If this is the baseline system, save the collocation information if (platform_index == 0) { writeVariableFloat(collocation_group_id, col_vid_lat, index, num_vals, latitude); writeVariableFloat(collocation_group_id, col_vid_lon, index, num_vals, longitude); writeVariableInteger(collocation_group_id, col_vid_date, index, num_vals, yyyymmdd); writeVariableInteger(collocation_group_id, col_vid_time, index, num_vals, hhmmss); } // Save the orbital node, data frame, scan line, fov and orbit if (buffer[11] == 1) writeVariableByte(nc_group_id, vid_node, index, num_vals, (char)0); else if (buffer[11] == 0) writeVariableByte(nc_group_id, vid_node, index, num_vals, (char)1); else writeVariableByte(nc_group_id, vid_node, index, num_vals, (char)-128); writeVariableShort(nc_group_id, vid_dataframe, index, num_vals, buffer[12]); writeVariableShort(nc_group_id, vid_scanline, index, num_vals, buffer[13]); writeVariableShort(nc_group_id, vid_fov, index, num_vals, buffer[14]); writeVariableInteger(nc_group_id, vid_orbit, index, num_vals, buffer[15]); // Zenith and azimuth angle if (buffer[21] != -32768) writeVariableFloat(nc_group_id, vid_solazimuth, index, num_vals, (buffer[21]/128.0)); else writeVariableFloat(nc_group_id, vid_solazimuth, index, num_vals, -32768.0); if (buffer[22] != -32768) writeVariableFloat(nc_group_id, vid_solzen, index, num_vals, (buffer[22]/128.0)); else writeVariableFloat(nc_group_id, vid_solzen, index, num_vals, -32768.0); if (buffer[23] != -32768) writeVariableFloat(nc_group_id, vid_satazimuth, index, num_vals, (buffer[23]/128.0)); else writeVariableFloat(nc_group_id, vid_satazimuth, index, num_vals, -32768.0); if (buffer[24] != -32768) writeVariableFloat(nc_group_id, vid_satzen, index, num_vals, (buffer[24]/128.0)); else writeVariableFloat(nc_group_id, vid_satzen, index, num_vals, -32768.0); // Surface values if (buffer[25] != -32768) writeVariableFloat(nc_group_id, vid_surf_height, index, num_vals, (buffer[25]/10.0)); else writeVariableFloat(nc_group_id, vid_surf_height, index, num_vals, -32768.0); if (buffer[26] != -32768) writeVariableFloat(nc_group_id, vid_land_frac, index, num_vals, (buffer[26]/1000.0)); else writeVariableFloat(nc_group_id, vid_land_frac, index, num_vals, -32768.0); if (buffer[27] != -32768) writeVariableFloat(nc_group_id, vid_surf_press, index, num_vals, (buffer[27]/10.0)); else writeVariableFloat(nc_group_id, vid_surf_press, index, num_vals, -32768.0); if (buffer[28] != -32768) writeVariableFloat(nc_group_id, vid_skin_temp, index, num_vals, (buffer[28]/64.0)); else writeVariableFloat(nc_group_id, vid_skin_temp, index, num_vals, -32768.0); // IR noise, prof diff and iterations if (buffer[29] != -32768) writeVariableFloat(nc_group_id, vid_noise, index, num_vals, (buffer[29]/10.0)); else writeVariableFloat(nc_group_id, vid_noise, index, num_vals, -32768.0); if (buffer[30] != -32768) writeVariableFloat(nc_group_id, vid_profdiff, index, num_vals, (buffer[30]/10.0)); else writeVariableFloat(nc_group_id, vid_profdiff, index, num_vals, -32768.0); if (buffer[31] != -32768) writeVariableFloat(nc_group_id, vid_iterations, index, num_vals, (buffer[31]/1.0)); else writeVariableFloat(nc_group_id, vid_iterations, index, num_vals, -32768.0); // Quality and other flags if (buffer[161] != -32768) writeVariableByte(nc_group_id, vid_qc, index, num_vals, (char)buffer[161]); else writeVariableByte(nc_group_id, vid_qc, index, num_vals, (char)-128); if (buffer[162] != -32768) writeVariableByte(nc_group_id, vid_cloud, index, num_vals, (char)buffer[162]); else writeVariableByte(nc_group_id, vid_cloud, index, num_vals, (char)-128); if (buffer[163] != -32768) writeVariableByte(nc_group_id, vid_ice, index, num_vals, (char)buffer[163]); else writeVariableByte(nc_group_id, vid_ice, index, num_vals, (char)-128); if (buffer[164] != -32768) writeVariableByte(nc_group_id, vid_rain, index, num_vals, (char)buffer[164]); else writeVariableByte(nc_group_id, vid_rain, index, num_vals, (char)-128); if (buffer[165] != -32768) writeVariableByte(nc_group_id, vid_cell, index, num_vals, (char)buffer[165]); else writeVariableByte(nc_group_id, vid_cell, index, num_vals, (char)-128); if (buffer[166] == 1) writeVariableByte(nc_group_id, vid_daynight, index, num_vals, (char)0); else if (buffer[166] == 0) writeVariableByte(nc_group_id, vid_daynight, index, num_vals, (char)1); else writeVariableByte(nc_group_id, vid_daynight, index, num_vals, (char)-128); if (buffer[167] != -32768) writeVariableByte(nc_group_id, vid_irmw_converge, index, num_vals, (char)buffer[167]); else writeVariableByte(nc_group_id, vid_irmw_converge, index, num_vals, (char)-128); if (buffer[168] != -32768) writeVariableByte(nc_group_id, vid_mw_converge, index, num_vals, (char)buffer[168]); else writeVariableByte(nc_group_id, vid_mw_converge, index, num_vals, (char)-128); if (buffer[169] != -32768) writeVariableByte(nc_group_id, vid_lte, index, num_vals, (char)buffer[169]); else writeVariableByte(nc_group_id, vid_lte, index, num_vals, (char)-128); if (buffer[170] != -32768) writeVariableByte(nc_group_id, vid_terrain, index, num_vals, (char)buffer[170]); else writeVariableByte(nc_group_id, vid_terrain, index, num_vals, (char)-128); // Chi square values if (buffer[685] != -32768) writeVariableFloat(nc_group_id, vid_irmw_chi, index, num_vals, (buffer[685]/100.0)); else writeVariableFloat(nc_group_id, vid_irmw_chi, index, num_vals, -32768.0); if (buffer[686] != -32768) writeVariableFloat(nc_group_id, vid_mw1_chi, index, num_vals, (buffer[686]/100.0)); else writeVariableFloat(nc_group_id, vid_mw1_chi, index, num_vals, -32768.0); if (buffer[687] != -32768) writeVariableFloat(nc_group_id, vid_mw2_chi, index, num_vals, (buffer[687]/100.0)); else writeVariableFloat(nc_group_id, vid_mw2_chi, index, num_vals, -32768.0); // Standard pressures and temperatures index_2D[0] = collocation_num; index_2D[1] = 0; num_vals_2D[0] = 1; num_vals_2D[1] = 42; for (n=0; n<42; n++) { if (buffer[33+n] == -32768) data42[n] = -32768.0; else if (n < 8) data42[n] = buffer[33+n] / 1000.0; else data42[n] = buffer[33+n] / 10.0; } writeArrayFloat(nc_group_id, vid_temppress, index_2D, num_vals_2D, data42); for (n=0; n<42; n++) { if (buffer[75+n] == -32768) data42[n] = -32768.0; else data42[n] = buffer[75+n] / 64.0; } writeArrayFloat(nc_group_id, vid_temp, index_2D, num_vals_2D, data42); // Standard pressures and water vapors index_2D[0] = collocation_num; index_2D[1] = 0; num_vals_2D[0] = 1; num_vals_2D[1] = 22; for (n=0; n<22; n++) { if (buffer[117+n] == -32768) data22[n] = -32768.0; else data22[n] = buffer[117+n] / 10.0; } writeArrayFloat(nc_group_id, vid_wvappress, index_2D, num_vals_2D, data22); for (n=0; n<22; n++) { if (buffer[139+n] == -32768) data22[n] = -32768.0; else data22[n] = (float)(exp(buffer[139+n]/1024.0)); //printf("WVAP: %d %f %d\n", n, data22[n], buffer[139+n]); } writeArrayFloat(nc_group_id, vid_wvap, index_2D, num_vals_2D, data22); // IR and MW Pressures index_2D[0] = collocation_num; index_2D[1] = 0; num_vals_2D[0] = 1; num_vals_2D[1] = 101; for (n=0; n<101; n++) { if (buffer[584+n] == -32768) data101[n] = -32768.0; else if (n < 14) data101[n] = buffer[584+n] / 10000.0; else if (n < 31) data101[n] = buffer[584+n] / 1000.0; else if (n < 65) data101[n] = buffer[584+n] / 100.0; else data101[n] = buffer[584+n] / 10.0; } writeArrayFloat(nc_group_id, vid_press, index_2D, num_vals_2D, data101); // Temperatures for (n=0; n<101; n++) { if (buffer[180+n] == -32768) data101[n] = -32768.0; else data101[n] = buffer[180+n] / 64.0; } writeArrayFloat(nc_group_id, vid_ir_temp, index_2D, num_vals_2D, data101); for (n=0; n<101; n++) { if (buffer[382+n] == -32768) data101[n] = -32768.0; else data101[n] = buffer[382+n] / 64.0; } writeArrayFloat(nc_group_id, vid_mw_temp, index_2D, num_vals_2D, data101); // Water vapor mixing ratio for (n=0; n<101; n++) { if (buffer[281+n] == -32768) data101[n] = -32768.0; else data101[n] = (float)(exp(buffer[281+n]/1024.0)); } writeArrayFloat(nc_group_id, vid_ir_wvap, index_2D, num_vals_2D, data101); for (n=0; n<101; n++) { if (buffer[483+n] == -32768) data101[n] = -32768.0; else data101[n] = (float)(exp(buffer[483+n]/1024.0)); } writeArrayFloat(nc_group_id, vid_mw_wvap, index_2D, num_vals_2D, data101); } // if (yyyymmdd == date_to_process... } // for (recnum=0... if (in != NULL) fclose(in); } // for (date_type=0... // Free the memory used by the buffer free(buffer); } // end of file