Skip to content

included reading of era5 files #316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HolgerPollyNet wants to merge 31 commits into
base: master
Choose a base branch
from
Open

included reading of era5 files #316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
3bb5b4a
A small smoothing was added into the meteorological data to avoid sha…
cristoferjimenez Feb 24, 2025
a6d0b94
Using the median shot number to recalculate the signal in photon coun…
cristoferjimenez Feb 24, 2025
8453b7e
Read all gdas files during the period and interpolate the molecular o…
cristoferjimenez Feb 24, 2025
bddc63a
Merge pull request #312 from PollyNET/processing_corrections2
HolgerPollyNet Mar 11, 2025
d459786
The co and cross component of the attenuated backscatter is computed …
cristoferjimenez Mar 14, 2025
e2b8ab2
added meteor sounding to available radio sonde readings
HolgerPollyNet Mar 14, 2025
a2713fc
Merge pull request #313 from PollyNET/Save_co_and_cross_beta
HolgerPollyNet Mar 25, 2025
793bc31
corrected Quasi Products version in donefile-writing and added zLim f…
dummyuser Apr 1, 2025
005c40c
Merge branch 'dev' into bug_fix_of_height_and_temp
HolgerPollyNet Apr 4, 2025
d9e3730
corrected height and range correction
HolgerPollyNet Apr 4, 2025
c526240
correction: calculation of Kelvin temperature
HolgerPollyNet Apr 4, 2025
fc246f1
Merge pull request #315 from PollyNET/bug_fix_of_height_and_temp
HolgerPollyNet Apr 7, 2025
7db6b80
warning for use of temp correction function
HolgerPollyNet May 14, 2025
3da6e8b
included ERA5 for reading meteorogical output
HolgerPollyNet May 15, 2025
4a31d5f
included era5
HolgerPollyNet May 16, 2025
5a7bbb3
cleaned up polly.global
HolgerPollyNet May 19, 2025
8d7fdc2
removed 1064 raman extinction derived from 607 channel
HolgerPollyNet May 20, 2025
81d7902
implemented PPC-processing for local device MARTHA
dummyuser Aug 20, 2025
36d5f51
added yLim for overlap-plotting in config
dummyuser Aug 22, 2025
78056df
Merge pull request #318 from PollyNET/implement_martha
ulysses78 Aug 22, 2025
759b58c
corrected concatinating script for martha if only one file was found
dummyuser Aug 25, 2025
5315568
updates related to issue #319
AthenaAugousta Sep 15, 2025
5db1224
Merge pull request #320 from PollyNET/fix_issue_319
HolgerPollyNet Sep 16, 2025
06d801e
bug fix wrt storing of uncertainty of LR
HolgerPollyNet Sep 29, 2025
4e49a75
Update pollyLR.m
HolgerPollyNet Sep 29, 2025
958a8ae
adding RR plots to the summary profile-plots as well as for the QC-plots
ulysses78 Oct 1, 2025
f224dd4
Merge pull request #321 from PollyNET/adding_RR-plots_to_summary-prof…
ulysses78 Oct 23, 2025
4f5d673
added flag no_merging to process without merging to picasso_g24.sh
dummyuser Oct 23, 2025
a49fe21
fix quasi depol
HolgerPollyNet Dec 3, 2025
2c2fc28
corrected bug in calculation of quasi par depol
HolgerPollyNet Dec 4, 2025
32d9615
Merge pull request #322 from PollyNET/depol-quasi-fix
HolgerPollyNet Dec 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
201 changes: 109 additions & 92 deletions lib/config/polly_global_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,38 @@
"flagMolDepolCali": false,
"flagTransCor": true,
"flagUseTheoreticalMDR": false,

"MWRFolder": "",
"flagUseLatestGDAS": true,

"dataFileFormat": "(?<year>\\d{4})_(?<month>\\d{2})_(?<day>\\d{2})_\\w*_(?<hour>\\d{2})_(?<minute>\\d{2})_(?<second>\\d{2})\\w*.nc",
"gdas1Site": "",

"meteorDataSource": "nc_cloudnet",
"meteo_folder": "/lacroshome/cloudnet/data/mindelo/calibrated/ecmwf",
"meteorDataSource": "gdas1",
"meteo_folder": "/data/level1a/model/gdas1/profiles",
"AERONETSite": "",
"radiosondeType": 1,
"radiosondeSitenum": 14430,
"IWV_instrument": "mwr_cloudnet",
"maxIWVTLag": 0.0833,
"MWRFolder": "/data/level1b/cloudnetpy/products/SITENAME",
"angstrexp_NR": 1.0,
"angstrexp": 1.0,

"max_height_bin": 3000,
"first_range_gate_indx": [251, 251, 251, 251, 251, 251, 251, 251, 252, 252, 252, 252, 252],
"cloudScreenMode": 1,
"maxSigSlope4FilterCloud": 0.7e5,
"maxSigSlope4FilterCloud_NR": 0.7e5,

"overlapCalMode": 2,
"overlapCorMode": 2,
"overlapSmoothBins": 8,

"intNProfiles": 120,
"minIntNProfiles": 60,

"init_depAng": 0,
"first_range_gate_indx": [251, 251, 251, 251, 251, 251, 251, 251, 252, 252, 252, 252, 252, 252, 252, 252, 252, 252],
"first_range_gate_height": 3.75,
"deltaT": 30,
"dtCorModeLabel": ["Polynomial coefficients stores in netcdf file", "nonparalyzable correction", "User defined polynomial cofficients"],
Expand All @@ -50,25 +71,33 @@
"bgCorRangeIndx": [10, 240],
"mask_SNRmin": [0.5, 0.5, 0.5, 0.01, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5],
"tempCorFunc": ["1", "1", "1", "1", "1", "1", "1", "1", "1", "1", "1", "1", "1"],
"init_depAng": 999,
"depol_cali_mode": 2,

"depol_cali_mode": 1,
"TR": [1.04, 500, 1, 1, 1.01, 1500, 1, 1, 1, 1, 1, 1, 1, 1, 500, 1, 1, 1, 1, 500, 1],
"G": [-999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999 ],
"H": [-999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999 ],
"K": [-999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999 ],
"voldepol_error_355": [-999, -999, -999 ],
"voldepol_error_532": [-999, -999, -999 ],
"voldepol_error_1064": [-999, -999, -999 ],
"depol_cal_time_fixed_p_start": [""],
"depol_cal_time_fixed_p_end": [""],
"depol_cal_time_fixed_m_start": [""],
"depol_cal_time_fixed_m_end": [""],
"maskDepCalAng": ["none", "none", "p", "p", "p", "p", "p", "p", "p", "p", "none", "n", "n", "n", "n", "n", "n", "n", "n"],

"depol_cal_minbin_532": 100,
"depol_cal_maxbin_532": 300,
"depol_cal_SNRmin_532": [1, 1, 1, 1],
"depol_cal_sigMax_532": [1500, 1500, 1500, 1500],
"depol_cal_sigMax_532": [2000, 2000, 2000, 2000],
"rel_std_dplus_532": 0.2,
"rel_std_dminus_532": 0.2,
"depol_cal_segmentLen_532": 40,
"depol_cal_smoothWin_532": 8,
"depol_cal_minbin_355": 100,
"depol_cal_maxbin_355": 300,
"depol_cal_SNRmin_355": [2, 2, 2, 2],
"depol_cal_sigMax_355": [1000, 1000, 1000, 1000],
"depol_cal_SNRmin_355": [1, 1, 1, 1],
"depol_cal_sigMax_355": [2000, 2000, 2000, 2000],
"rel_std_dplus_355": 0.2,
"rel_std_dminus_355": 0.2,
"depol_cal_segmentLen_355": 40,
Expand All @@ -81,73 +110,57 @@
"rel_std_dminus_1064": 0.2,
"depol_cal_segmentLen_1064": 40,
"depol_cal_smoothWin_1064": 8,
"isFR": [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
"isNR": [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0],
"is532nm": [0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0],
"isRR": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"is355nm": [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],
"is1064nm": [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],
"isTot": [1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0],
"isCross": [0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0],
"isParallel": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"is387nm": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
"is407nm": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"is607nm": [0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0],
"is1058nm": [0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0],
"isFR": [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0],
"isNR": [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"is532nm": [0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"is355nm": [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"is1064nm": [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0],
"isTot": [1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"isCross": [0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0],
"isRR": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0],
"isParallel": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"is387nm": [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"is407nm": [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"is607nm": [0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
"is1058nm": [],
"minSNR_4_sigNorm": [],
"heightFullOverlap": [900, 900, 900, 900, 800, 800, 800, 800, 150, 150, 150, 150, 150, 800, 800,800, 900, 900, 900, 900, 800],

"channelTags": [],
"channelTag": ["FR-total-355 nm", "FR-cross-355 nm", "FR-387 nm", "FR-407 nm", "FR-total-532 nm", "FR-cross-532 nm", "FR-607 nm", "FR-total-1064 nm", "NR-total-532 nm", "NR-607 nm", "NR-total-355 nm", "NR-387 nm", "unknown"],
"channelTag": ["FR-total-355 nm", "FR-cross-355 nm", "FR-387 nm", "FR-407 nm", "FR-total-532 nm", "FR-cross-532 nm", "FR-607 nm", "FR-total-1064 nm", "NR-total-532 nm", "NR-607 nm", "NR-total-355 nm", "NR-387 nm", "DFOV","1058","1064s","none", "none", "none", "none", "none", "none"],

"minPC_fog": 60,
"TR": [0.898, 1086, 1, 1, 1.45, 778.8, 1, 1, 1, 1, 1, 1, 1],
"G": [-999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999 ],
"H": [-999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999 ],
"K": [-999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999, -999 ],
"voldepol_error_355": [-999, -999, -999 ],
"voldepol_error_532": [-999, -999, -999 ],
"voldepol_error_1064": [-999, -999, -999],
"overlapCalMode": 1,
"overlapCorMode": 3,
"overlapSmoothBins": 8,

"saturate_thresh": 100,
"heightFullOverlap": [500, 500, 500, 500, 500, 500, 500, 500, 150, 150, 150, 150, 150],
"minSNR_4_sigNorm": [10],
"cloudScreenMode": 1,
"maxSigSlope4FilterCloud": 3e6,
"maxSigSlope4FilterCloud_NR": 4e5,
"intNProfiles": 120,
"minIntNProfiles": 90,
"meteorDataSource": "gdas1",
"flagUseLatestGDAS": true,
"radiosondeType": 1,
"radiosondeSitenum": 14430,
"minDecomLogDist355": 0.2,
"minDecomLogDist532": 0.2,
"minDecomLogDist1064": 0.2,
"maxDecomHeight355": 10000,
"maxDecomHeight532": 10000,
"maxDecomHeight1064": 10000,
"maxDecomThickness355": 1500,
"maxDecomThickness532": 1500,
"maxDecomThickness1064": 1500,
"decomSmoothWin355": 40,

"minDecomLogDist355": 2,
"minDecomLogDist532": 2,
"minDecomLogDist1064": 2,
"maxDecomHeight355": 18000,
"maxDecomHeight532": 18000,
"maxDecomHeight1064": 25000,
"maxDecomThickness355": 6000,
"maxDecomThickness532": 3000,
"maxDecomThickness1064": 2000,
"decomSmoothWin355": 80,
"decomSmoothWin532": 40,
"decomSmoothWin1064": 120,
"minRefThickness355": 700,
"minRefThickness532": 700,
"minRefThickness1064": 700,
"minRefDeltaExt355": 1,
"minRefDeltaExt532": 1,
"minRefDeltaExt1064": 1,
"decomSmoothWin1064": 30,
"minRefThickness355": 1000,
"minRefThickness532": 600,
"minRefThickness1064": 300,
"minRefDeltaExt355": 11,
"minRefDeltaExt532": 11,
"minRefDeltaExt1064":11,
"refH_FR_355": [],
"refH_FR_532": [],
"refH_FR_1064": [],
"refH_NR_355": [2500, 3000],
"refH_NR_532": [2500, 3000],
"minRefSNR355": 10,
"minRefSNR355": 5,
"minRefSNR532": 5,
"minRefSNR1064": 3,
"minRefSNR_NR_355": 10,
"minRefSNR_NR_532": 5,
"minRefSNR1064": 0.5,
"minRefSNR_NR_355": 3,
"minRefSNR_NR_532": 3,
"LR355": 50,
"LR532": 50,
"LR1064": 50,
Expand All @@ -170,30 +183,28 @@
"minLRConstrainFernald": 1,
"maxLRConstrainFernald": 150,
"minDeltaAOD": 0.01,
"minRamanRefSNR355": 50,
"minRamanRefSNR532": 20,
"minRamanRefSNR1064": 10,
"minRamanRefSNR387": 40,
"minRamanRefSNR607": 20,
"minRamanRefSNR_NR_355": 50,
"minRamanRefSNR_NR_532": 20,
"minRamanRefSNR_NR_387": 40,
"minRamanRefSNR_NR_607": 20,
"minRamanRefSNR355": 5,
"minRamanRefSNR532": 4,
"minRamanRefSNR1064": 0.5,
"minRamanRefSNR387": 3,
"minRamanRefSNR607": 2,
"minRamanRefSNR_NR_355": 2,
"minRamanRefSNR_NR_532": 4,
"minRamanRefSNR_NR_387": 2,
"minRamanRefSNR_NR_607": 2,
"min_RR_RefSNR1058": 0,
"angstrexp": 0.9,
"angstrexp_NR": 0.9,

"LCMeanWindow": 50,
"LCMeanMinIndx": 70,
"LCMeanMinIndx": 150,
"LCMeanMaxIndx": 1000,
"LCCalibrationStatus": ["none", "Klett", "Raman", "Defaults", "History"],
"quasi_smooth_h": [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8],
"quasi_smooth_t": [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],
"IWV_instrument": "AERONET",
"maxIWVTLag": 0.0833,
"quasi_smooth_h": [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],
"quasi_smooth_t": [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10],

"tTwilight": 0.0347,
"hWVCaliBase": 120,
"hWVCaliTop": 8000,
"minSNRWVCali": 10,
"minSNRWVCali": 1,

"clear_thres_par_beta_1064": 1e-8,
"turbid_thres_par_beta_1064": 2e-7,
Expand Down Expand Up @@ -221,7 +232,7 @@
"xLim_Profi_WVMR": [0, 50],
"xLim_Profi_RCS": [0.3, 1000],
"xLim_Profi_LR": [0, 120],
"xLim_Profi_AE": [-1,2],
"xLim_Profi_AE": [-1,3],
"xLim_beta_532_Poliphon": [0, 10],
"yLim_beta_532_Poliphon": [0, 8000],
"yLim_LC_355": [0, 1e14],
Expand All @@ -231,6 +242,9 @@
"yLim_LC_607": [0, 3e14],
"yLim_LC_355_NR": [0, 1e13],
"yLim_LC_532_NR": [0, 2e13],
"yLim_LC_ratio_355_387": [0, 2],
"yLim_LC_ratio_532_607": [0, 2],

"yLim_WVConst": [0, 40],
"yLim_FR_RCS": [0, 20000],
"yLim_NR_RCS": [0, 3000],
Expand All @@ -245,21 +259,24 @@
"yLim_Profi_DR": [0, 20000],
"yLim_Profi_Bsc": [0, 20000],
"yLim_Profi_WV_RH": [0, 7000],
"yLim_LC_ratio_355_387": [0, 2],
"yLim_LC_ratio_532_607": [0, 2],
"yLim_all_profiles_high_range": [0, 18000],
"yLim_all_profiles_low_range": [0, 5000],


"yLim_depolConst_355": [0, 50],
"yLim_depolConst_532": [0, 50],
"yLim_depolConst_1064": [0, 50],
"yLim_depolConst_1064": [0, 10],
"yLim_cloudinfo": [0, 2000],
"yLim_all_profiles_high_range": [0, 18000],
"yLim_all_profiles_low_range": [0, 5000],
"yLim_overlap": [0, 3000],

"zLim_att_beta_355": [0, 15],
"zLim_att_beta_532": [0, 5],
"zLim_att_beta_1064": [0, 2],
"zLim_quasi_beta_355": [0, 10],
"zLim_quasi_beta_532": [0, 6],
"zLim_quasi_beta_1064": [0, 3],
"zLim_quasi_beta_355": [0, 10e-6],
"zLim_quasi_beta_532": [0, 6e-6],
"zLim_quasi_beta_1064": [0, 3e-6],
"zLim_quasi_Par_DR_532": [0, 0.4],
"zLim_quasi_ANG": [-1,3],
"zLim_FR_RCS_355": [1e-2, 30],
"zLim_FR_RCS_387": [1e-2, 30],
"zLim_FR_RCS_407": [1e-2, 35],
Expand Down Expand Up @@ -290,12 +307,12 @@
"Data_Originator_email": "",
"comment": "",

"partnerLabel": "",
"calibrationDB": "polly_calibration.db",
"logbookFile": "",
"logbookPath": "",
"logbookFileName": "",
"radiosondeFolder": "",
"calibrationDB": "polly_calibration.db",
"imgFormat": "png",
"partnerLabel": "",
"prodSaveList": ["overlap", "aerProfFR", "aerProfNR", "aerProfOC", "aerAttBetaFR", "aerAttBetaOC", "aerAttBetaNR", "WVMR_RH", "volDepol", "quasiV1", "quasiV2", "TC", "TCV2", "cloudinfo", "poliphon_one","RCS"]
}
Loading