;-------------------------------------------------------------------------------------------- ; probabilistic mask and phase tables for use in the clavrx naive bayesian ; ; Author: Andrew Heidinger ; Date: December 10, 2017 ; ; uses calipso_matchups that include clavrx data ; ;-------------------------------------------------------------------------------------------- @push pro make_mask_phase_training missing = -999.0 training_data_vector = {sensor: "snpp", $ time_diff: 0.0, $ solzen: 0.0, $ zen: 0.0, $ scatzen: 0.0, $ glintzen: 0.0, $ year: 0, $ jday: 0, $ lat: 0.0, $ lon: 0.0, $ zsfc: 0.0, $ tsfc_model: 0.0, $ sst_back_uni: 0.0, $ land_class: 0, $ snow_class: 0, $ sfc_type: 0, $ coast_mask: 0, $ glint_mask: 0, $ cod_lidar: 0.0, $ codi_lidar: 0.0, $ codw_lidar: 0.0, $ phase_lidar: 0, $ multilayer_flag_lidar: 0, $ num_cld_layers_lidar: 0, $ num_cld_layers_min_lidar: 0, $ num_cld_layers_max_lidar: 0, $ cld_frac_lidar: 0.0, $ ec_lidar: 0.0, $ tc_lidar: 0.0, $ pc_lidar: 0.0, $ zc_eff_lidar: 0.0, $ zc_geo_lidar: 0.0, $ tc_opaque_cloud: 0.0, $ zc_opaque_cloud: 0.0, $ logt11_std: 0.0, $ t11_std: 0.0, $ ref_047um: 0.0, $ ref_065um: 0.0, $ ref_086um: 0.0, $ ref_138um: 0.0, $ ref_160um: 0.0, $ ref_375um: 0.0, $ bt_375um: 0.0, $ bt_67um: 0.0, $ bt_73um: 0.0, $ bt_85um: 0.0, $ bt_11um: 0.0, $ bt_12um: 0.0, $ bt_133um: 0.0, $ ref_065um_clr: 0.0, $ ref_375um_clr: 0.0, $ bt_375um_clr: 0.0, $ bt_67um_clr: 0.0, $ bt_73um_clr: 0.0, $ bt_85um_clr: 0.0, $ bt_11um_clr: 0.0, $ bt_12um_clr: 0.0, $ bt_133um_clr: 0.0, $ ref_065um_min_3x3: 0.0, $ ref_065um_min_sub: 0.0, $ ref_065um_max_sub: 0.0, $ ref_086um_min_sub: 0.0, $ ref_086um_max_sub: 0.0, $ bt_11um_min_sub: 0.0, $ bt_11um_max_sub: 0.0, $ opd_mask_065um: 0.0, $ opd_mask_138um: 0.0, $ emiss_tropo: 0.0, $ ems_375um: 0.0, $ ems_375um_clr: 0.0, $ sfc_emiss_375um: 0.0, $ logbt_11um_std_3x3: 0.0, $ bt_11um_std_3x3: 0.0, $ bt_11um_max_3x3: 0.0, $ bt_11um_bt_67um_covar: 0.0, $ btd_11um_12um_bt_11um_max_3x3: 0.0, $ bayes_mask_sfc_type: 0, $ beta_11_67_tropo: 0.0, $ beta_11_85_tropo: 0.0, $ beta_11_12_tropo: 0.0, $ beta_11_133_tropo: 0.0, $ tpw: 0.0, $ cld_frac: 0.0, $ cld_frac_uncer: 0.0, $ cld_prob: 0.0, $ cld_mask: 0.0, $ cld_type: 0.0, $ cld_mask_aux: 0.0, $ cld_type_aux: 0.0} ;------------------------------ ; read in data ;------------------------------- path = './calipso_matchup_5p1km_clavrx/' files = file_search(path,'*.calipso.hdf') nfiles = n_elements(files) n = nfiles ;n = 1000 for ifile = 0, n - 1 do begin print,'-----------------------------------------------' print,'---------------------------------------------------------------------------------' print,'-----------------------------------------------' print,'--------------------' print, ifile+1, ' of ', n print,'--------------------' print,'-----------------------------------------------' print,'---------------------------------------------------------------------------------' print,'-----------------------------------------------' xlat = read_standard_hdf(files[ifile],'closest_calipso_latitude') p = n_elements(xlat) no_data_r4 = make_array(p,/FLOAT, VALUE=missing) no_data_i2 = make_array(p,/INT, VALUE=fix(missing)) x = replicate(training_data_vector, p) ;-- calipso r4 read_r4, files[ifile], 'closest_calipso_time_difference', f, no_data_r4 & x.time_diff = f read_r4, files[ifile], 'closest_calipso_longitude', f, no_data_r4 & x.lon = f read_r4, files[ifile], 'closest_calipso_latitude', f, no_data_r4 & x.lat = f read_r4, files[ifile], 'closest_calipso_cod', f, no_data_r4 & x.cod_lidar = f read_r4, files[ifile], 'closest_calipso_cod_ice', f, no_data_r4 & x.codi_lidar = f read_r4, files[ifile], 'closest_calipso_cod_water', f, no_data_r4 & x.codw_lidar = f read_r4, files[ifile], 'closest_calipso_cloud_fraction_15km', f, no_data_r4 & x.cld_frac_lidar = f read_r4, files[ifile], 'closest_calipso_top_alt_eff_geo', f, no_data_r4 & x.zc_eff_lidar = f read_r4, files[ifile], 'closest_calipso_top_pres_eff_geo', f, no_data_r4 & x.zc_geo_lidar = f prof_temp = read_standard_hdf(files[ifile],'closest_calipso_cloud_mid_temperature') x.tc_lidar = reform(prof_temp[0,*]) read_r4, files[ifile], 'closest_calipso_top_pres_eff_geo', f, no_data_r4 & x.zc_geo_lidar = f ;-- calipso i2 read_r4, files[ifile], 'closest_calipso_cloud_phase_top_layer', f, no_data_i2 & x.phase_lidar = f read_r4, files[ifile], 'closest_calipso_multilayer_flag', f, no_data_i2 & x.multilayer_flag_lidar = f read_r4, files[ifile], 'closest_calipso_number_of_cloud_layers_5km', f, no_data_i2 & x.num_cld_layers_lidar = f ;-- clavrx r4 read_r4, files[ifile], 'cloud_probability', f, no_data_r4 & x.cld_prob = f read_r4, files[ifile], 'cloud_fraction', f, no_data_r4 & x.cld_frac = f read_r4, files[ifile], 'cloud_fraction_uncertainty', f, no_data_r4 & x.cld_frac_uncer = f read_r4, files[ifile], 'solar_zenith_angle', f, no_data_r4 & x.solzen = f read_r4, files[ifile], 'sensor_zenith_angle', f, no_data_r4 & x.zen = f read_r4, files[ifile], 'scattering_angle', f, no_data_r4 & x.scatzen = f read_r4, files[ifile], 'glint_zenith_angle', f, no_data_r4 & x.glintzen = f read_r4, files[ifile], 'closest_calipso_dem_elev',f, no_data_r4 & x.zsfc = f read_r4, files[ifile], 'surface_temperature_nwp',f, no_data_r4 & x.tsfc_model = f read_r4, files[ifile], 'sst_background_uni_3x3', f, no_data_r4 & x.sst_back_uni = f read_r4, files[ifile], 'emiss_sfc_3_75um_nom', f, no_data_r4 & x.sfc_emiss_375um = f read_r4, files[ifile], 'refl_0_47um_nom', f, no_data_r4 & x.ref_047um = f read_r4, files[ifile], 'refl_0_65um_nom', f, no_data_r4 & x.ref_065um = f read_r4, files[ifile], 'refl_0_86um_nom', f, no_data_r4 & x.ref_086um = f read_r4, files[ifile], 'refl_1_38um_nom', f, no_data_r4 & x.ref_138um = f read_r4, files[ifile], 'refl_1_60um_nom', f, no_data_r4 & x.ref_160um = f read_r4, files[ifile], 'temp_3_75um_nom', f, no_data_r4 & x.bt_375um = f read_r4, files[ifile], 'temp_6_7um_nom', f, no_data_r4 & x.bt_67um = f read_r4, files[ifile], 'temp_7_3um_nom', f, no_data_r4 & x.bt_73um = f read_r4, files[ifile], 'temp_8_5um_nom', f, no_data_r4 & x.bt_85um = f read_r4, files[ifile], 'temp_11_0um_nom', f, no_data_r4 & x.bt_11um = f read_r4, files[ifile], 'temp_12_0um_nom', f, no_data_r4 & x.bt_12um = f read_r4, files[ifile], 'temp_13_3um_nom', f, no_data_r4 & x.bt_133um = f read_r4, files[ifile], 'emiss_3_75um_nom', f, no_data_r4 & x.ems_375um = f read_r4, files[ifile], 'emiss_tropo_11_0um_nom', f, no_data_r4 & x.emiss_tropo = f read_r4, files[ifile], 'refl_0_65um_nom_clear_sky', f, no_data_r4 & x.ref_065um_clr = f read_r4, files[ifile], 'temp_3_75um_nom_clear_sky', f, no_data_r4 & x.bt_375um_clr = f read_r4, files[ifile], 'temp_6_7um_nom_clear_sky', f, no_data_r4 & x.bt_67um_clr = f read_r4, files[ifile], 'temp_8_5um_nom_clear_sky', f, no_data_r4 & x.bt_85um_clr = f read_r4, files[ifile], 'temp_11_0um_nom_clear_sky', f, no_data_r4 & x.bt_11um_clr = f read_r4, files[ifile], 'temp_12_0um_nom_clear_sky', f, no_data_r4 & x.bt_12um_clr = f read_r4, files[ifile], 'temp_13_3um_nom_clear_sky', f, no_data_r4 & x.bt_133um_clr = f read_r4, files[ifile], 'emiss_3_75um_nom_clear', f, no_data_r4 & x.ems_375um_clr = f read_r4, files[ifile], 'refl_0_65um_nom_min_3x3', f, no_data_r4 & x.ref_065um_min_3x3 = f read_r4, files[ifile], 'temp_11_0um_nom_max_3x3', f, no_data_r4 & x.bt_11um_max_3x3 = f read_r4, files[ifile], 'temp_11_0um_nom_stddev_3x3', f, no_data_r4 & x.bt_11um_std_3x3 = f read_r4, files[ifile], 'temp_11um_vs_67um_covar_5x5', f, no_data_r4 & x.bt_11um_bt_67um_covar = f read_r4, files[ifile], 'temp_11_0um_nom_stddev_3x3', f, no_data_r4 & x.logbt_11um_std_3x3 = f read_r4, files[ifile], 'diff_ch31_ch32_Bt_ch31_max_3x3', f, no_data_r4 & x.btd_11um_12um_bt_11um_max_3x3 = f read_r4, files[ifile], 'refl_0_65um_nom_min_sub', f, no_data_r4 & x.ref_065um_min_sub = f read_r4, files[ifile], 'refl_0_65um_nom_max_sub', f, no_data_r4 & x.ref_065um_max_sub = f read_r4, files[ifile], 'refl_0_86um_nom_min_sub', f, no_data_r4 & x.ref_086um_min_sub = f read_r4, files[ifile], 'refl_0_86um_nom_max_sub', f, no_data_r4 & x.ref_086um_max_sub = f read_r4, files[ifile], 'temp_11_0um_nom_min_sub', f, no_data_r4 & x.bt_11um_min_sub = f read_r4, files[ifile], 'temp_11_0um_nom_max_sub', f, no_data_r4 & x.bt_11um_max_sub = f read_r4, files[ifile], 'cld_opd_mask_0_65um_nom',f, no_data_r4 & x.opd_mask_065um = f read_r4, files[ifile], 'cld_opd_mask_1_38um_nom',f, no_data_r4 & x.opd_mask_138um = f read_r4, files[ifile], 'cld_temp_opaque', f, no_data_r4 & x.tc_opaque_cloud = f read_r4, files[ifile], 'cld_height_opaque', f, no_data_r4 & x.zc_opaque_cloud = f read_r4, files[ifile], 'beta_11um_12um_tropopause', f, no_data_r4 & x.beta_11_12_tropo = f read_r4, files[ifile], 'beta_11um_85um_tropopause', f, no_data_r4 & x.beta_11_85_tropo = f read_r4, files[ifile], 'beta_11um_133um_tropopause', f, no_data_r4 & x.beta_11_133_tropo = f ;-- i2 read_i2, files[ifile], 'cloud_mask', f, no_data_i2 & x.cld_mask = f read_i2, files[ifile], 'cloud_type', f, no_data_i2 & x.cld_type = f read_i2, files[ifile], 'cloud_mask_aux', f, no_data_i2 & x.cld_mask_aux = f read_i2, files[ifile], 'cloud_type_aux', f, no_data_i2 & x.cld_type_aux = f read_i2, files[ifile], 'land_class', f, no_data_i2 & x.land_class = f read_i2, files[ifile], 'snow_class', f, no_data_i2 & x.snow_class = f read_i2, files[ifile], 'coast_mask', f, no_data_i2 & x.coast_mask = f read_i2, files[ifile], 'glint_mask', f, no_data_i2 & x.glint_mask = f read_i2, files[ifile], 'bayes_mask_sfc_type',f, no_data_i2 & x.bayes_mask_sfc_type = f read_i2, files[ifile], 'year',f, no_data_i2 & x.year = f read_i2, files[ifile], 'doy',f, no_data_i2 & x.jday = f read_i2, files[ifile], 'surface_type', f, no_data_i2 & x.sfc_type = f push, training_data, x endfor ;--- make log of stddev idx = where(training_data.logbt_11um_std_3x3 le 0.0,cc) if (cc gt 0) then training_data[idx].logbt_11um_std_3x3 = missing idx = where(training_data.logt11_std gt 0.0,cc) if (cc gt 0) then training_data[idx].logbt_11um_std_3x3 = alog10(training_data[idx].logbt_11um_std_3x3) training_creation_time = timestamp() ;training_data = CREATE_STRUCT('creation_time', training_creation_time, training_data) save, file='phase_training.sav', training_data, training_creation_time end ;------------------------------------------------------------------------------------------------------ ; ;------------------------------------------------------------------------------------------------------ pro read_r4, file, sds_name, output, missing_values output = read_standard_hdf(file,sds_name,/quiet) if (n_elements(output) eq 1) then begin print, 'no data for ', sds_name output = missing_values endif end ;------------------------------------------------------------------------------------------------------ ; ;------------------------------------------------------------------------------------------------------ pro read_i2, file, sds_name, output, missing_values output = read_standard_hdf(file,sds_name,/quiet) if (n_elements(output) eq 1) then begin print, 'no data for ', sds_name output = missing_values endif end