Explainable, XGBoost Model Development¶
import os
import numpy as np
import xarray as xr
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import getpass
import act# Note - recommended ARM Live token to be set as an environmental variable
# Check for env variables, otherwise set your username and token here!
if "ARM_USERNAME" in os.environ:
arm_username = os.getenv("ARM_USERNAME")
else:
arm_username = input("Enter ARM username: ").strip()
if "ARM_TOKEN" in os.environ:
arm_token = os.getenv("ARM_TOKEN")
else:
arm_token = getpass.getpass("Enter ARM token (hidden): ").strip()
startdate = '2019-11-01'
enddate = '2019-12-31'
datastream1 = 'sgpmergedsmpsapsE13.c1'
datastream2 = 'sgpacsmcdceE13.c2'
datastream3 = 'sgpaoppsap1flynn1mE13.c1'
datastream4 = 'sgpaosmetE13.a1'
datastream5 = 'sgpceil10mC1.b1' # <-------- Not using this in ML modelling
datastream6 = 'sgpaosccn2colaavgE13.b1'
result_datastream1_met = act.discovery.download_arm_data(arm_username, arm_token, datastream1, startdate, enddate)
result_datastream2_met = act.discovery.download_arm_data(arm_username, arm_token, datastream2, startdate, enddate)
result_datastream3_met = act.discovery.download_arm_data(arm_username, arm_token, datastream3, startdate, enddate)
result_datastream4_met = act.discovery.download_arm_data(arm_username, arm_token, datastream4, startdate, enddate)
result_datastream5_met = act.discovery.download_arm_data(arm_username, arm_token, datastream5, startdate, enddate)
result_datastream6_met = act.discovery.download_arm_data(arm_username, arm_token, datastream6, startdate, enddate)Enter ARM username: abhigyan
Enter ARM token (hidden): ········
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191101.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191102.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191103.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191104.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191105.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191106.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191107.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191108.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191109.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191110.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191111.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191112.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191113.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191114.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191115.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191116.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191117.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191118.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191119.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191120.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191121.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191122.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191123.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191124.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191125.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191126.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191127.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191128.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191129.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191130.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191201.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191202.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191203.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191204.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191205.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191206.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191207.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191208.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191209.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191210.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191211.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191212.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191213.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191214.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191215.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191216.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191217.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191218.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191219.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191220.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191221.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191222.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191223.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191224.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191225.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191226.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191227.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191228.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191229.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191230.000000.nc
[DOWNLOADING] sgpmergedsmpsapsE13.c1.20191231.000000.nc
If you use these data to prepare a publication, please cite:
Shilling, J., & Levin, M. merged size distribution from SMPS and APS
(MERGEDSMPSAPS), 2019-11-01 to 2019-12-31, Southern Great Plains (SGP), Lamont,
OK (Extended and Co-located with C1) (E13). Atmospheric Radiation Measurement
(ARM) User Facility. https://doi.org/10.5439/1871375
[DOWNLOADING] sgpacsmcdceE13.c2.20191101.000741.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191102.000914.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191103.000937.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191104.000034.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191105.000038.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191106.000218.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191107.000331.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191108.000710.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191109.000231.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191110.000413.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191111.000740.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191112.000914.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191113.000048.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191114.000227.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191115.000402.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191116.000900.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191117.000057.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191118.000413.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191119.000907.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191120.000725.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191121.000845.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191122.000339.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191123.000209.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191124.000340.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191125.000534.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191126.000851.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191127.000749.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191128.000625.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191129.000222.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191130.000710.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191201.000845.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191202.000511.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191203.000214.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191204.001000.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191205.000919.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191206.000608.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191207.001706.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191208.001254.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191209.000628.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191210.000612.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191211.000956.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191212.002018.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191213.000853.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191214.000127.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191215.002041.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191216.001234.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191217.000135.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191218.002136.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191219.001223.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191220.000619.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191221.000113.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191222.000134.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191223.000106.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191224.002357.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191225.001703.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191226.000921.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191227.000236.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191228.000114.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191229.000131.nc
[DOWNLOADING] sgpacsmcdceE13.c2.20191230.000129.nc
If you use these data to prepare a publication, please cite:
Zawadowicz, M., Howie, J., Shilling, J., & Levin, M. ACSM, corrected for
composition-dependent collection efficiency (ACSMCDCE), 2019-11-01 to
2019-12-31, Southern Great Plains (SGP), Lamont, OK (Extended and Co-located
with C1) (E13). Atmospheric Radiation Measurement (ARM) User Facility.
https://doi.org/10.5439/1763029
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191101.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191102.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191103.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191104.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191105.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191106.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191107.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191108.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191109.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191110.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191111.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191112.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191113.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191114.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191115.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191116.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191117.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191118.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191119.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191120.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191121.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191122.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191123.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191124.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191125.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191126.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191127.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191128.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191129.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191130.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191201.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191202.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191203.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191204.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191205.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191206.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191207.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191208.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191209.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191210.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191211.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191212.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191213.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191214.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191215.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191216.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191217.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191218.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191219.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191220.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191221.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191222.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191223.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191224.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191225.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191226.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191227.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191228.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191229.000030.nc
[DOWNLOADING] sgpaoppsap1flynn1mE13.c1.20191230.000030.nc
If you use these data to prepare a publication, please cite:
Koontz, A., Flynn, C., Shilling, J., & Flynn, C. Aerosol Optical Properties
(AOPPSAP1FLYNN1M), 2019-11-01 to 2019-12-31, Southern Great Plains (SGP),
Lamont, OK (Extended and Co-located with C1) (E13). Atmospheric Radiation
Measurement (ARM) User Facility. https://doi.org/10.5439/1369240
[DOWNLOADING] sgpaosmetE13.a1.20191101.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191102.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191103.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191104.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191105.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191106.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191107.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191108.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191109.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191110.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191111.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191112.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191113.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191114.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191115.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191116.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191117.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191118.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191119.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191120.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191121.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191122.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191123.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191124.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191125.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191126.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191127.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191128.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191129.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191130.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191201.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191202.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191203.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191204.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191205.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191206.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191207.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191208.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191209.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191210.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191211.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191212.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191213.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191214.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191215.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191216.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191217.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191218.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191219.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191220.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191221.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191222.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191223.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191224.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191225.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191226.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191227.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191228.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191229.000000.nc
[DOWNLOADING] sgpaosmetE13.a1.20191230.000000.nc
If you use these data to prepare a publication, please cite:
Kyrouac, J., Springston, S., & Tuftedal, M. Meteorological Measurements
associated with the Aerosol Observing System (AOSMET), 2019-11-01 to 2019-12-31,
Southern Great Plains (SGP), Lamont, OK (Extended and Co-located with C1) (E13).
Atmospheric Radiation Measurement (ARM) User Facility.
https://doi.org/10.5439/1984920
[DOWNLOADING] sgpceil10mC1.b1.20191101.000001.nc
[DOWNLOADING] sgpceil10mC1.b1.20191102.000014.nc
[DOWNLOADING] sgpceil10mC1.b1.20191103.000012.nc
[DOWNLOADING] sgpceil10mC1.b1.20191104.000009.nc
[DOWNLOADING] sgpceil10mC1.b1.20191105.000008.nc
[DOWNLOADING] sgpceil10mC1.b1.20191106.000005.nc
[DOWNLOADING] sgpceil10mC1.b1.20191107.000003.nc
[DOWNLOADING] sgpceil10mC1.b1.20191108.000000.nc
[DOWNLOADING] sgpceil10mC1.b1.20191109.000014.nc
[DOWNLOADING] sgpceil10mC1.b1.20191110.000011.nc
[DOWNLOADING] sgpceil10mC1.b1.20191111.000009.nc
[DOWNLOADING] sgpceil10mC1.b1.20191112.000006.nc
[DOWNLOADING] sgpceil10mC1.b1.20191113.000003.nc
[DOWNLOADING] sgpceil10mC1.b1.20191114.000001.nc
[DOWNLOADING] sgpceil10mC1.b1.20191115.000015.nc
[DOWNLOADING] sgpceil10mC1.b1.20191116.000012.nc
[DOWNLOADING] sgpceil10mC1.b1.20191117.000010.nc
[DOWNLOADING] sgpceil10mC1.b1.20191118.000007.nc
[DOWNLOADING] sgpceil10mC1.b1.20191119.000005.nc
[DOWNLOADING] sgpceil10mC1.b1.20191120.000003.nc
[DOWNLOADING] sgpceil10mC1.b1.20191121.000000.nc
[DOWNLOADING] sgpceil10mC1.b1.20191122.000014.nc
[DOWNLOADING] sgpceil10mC1.b1.20191123.000012.nc
[DOWNLOADING] sgpceil10mC1.b1.20191124.000009.nc
[DOWNLOADING] sgpceil10mC1.b1.20191125.000006.nc
[DOWNLOADING] sgpceil10mC1.b1.20191126.000004.nc
[DOWNLOADING] sgpceil10mC1.b1.20191127.000002.nc
[DOWNLOADING] sgpceil10mC1.b1.20191128.000016.nc
[DOWNLOADING] sgpceil10mC1.b1.20191129.000013.nc
[DOWNLOADING] sgpceil10mC1.b1.20191130.000011.nc
[DOWNLOADING] sgpceil10mC1.b1.20191201.000009.nc
[DOWNLOADING] sgpceil10mC1.b1.20191202.000006.nc
[DOWNLOADING] sgpceil10mC1.b1.20191203.000004.nc
[DOWNLOADING] sgpceil10mC1.b1.20191204.000001.nc
[DOWNLOADING] sgpceil10mC1.b1.20191205.000014.nc
[DOWNLOADING] sgpceil10mC1.b1.20191206.000012.nc
[DOWNLOADING] sgpceil10mC1.b1.20191207.000009.nc
[DOWNLOADING] sgpceil10mC1.b1.20191208.000008.nc
[DOWNLOADING] sgpceil10mC1.b1.20191209.000005.nc
[DOWNLOADING] sgpceil10mC1.b1.20191210.000002.nc
[DOWNLOADING] sgpceil10mC1.b1.20191211.000000.nc
[DOWNLOADING] sgpceil10mC1.b1.20191212.000014.nc
[DOWNLOADING] sgpceil10mC1.b1.20191213.000011.nc
[DOWNLOADING] sgpceil10mC1.b1.20191214.000008.nc
[DOWNLOADING] sgpceil10mC1.b1.20191215.000006.nc
[DOWNLOADING] sgpceil10mC1.b1.20191216.000004.nc
[DOWNLOADING] sgpceil10mC1.b1.20191217.000001.nc
[DOWNLOADING] sgpceil10mC1.b1.20191218.000014.nc
[DOWNLOADING] sgpceil10mC1.b1.20191219.000012.nc
[DOWNLOADING] sgpceil10mC1.b1.20191220.000009.nc
[DOWNLOADING] sgpceil10mC1.b1.20191221.000007.nc
[DOWNLOADING] sgpceil10mC1.b1.20191222.000005.nc
[DOWNLOADING] sgpceil10mC1.b1.20191223.000002.nc
[DOWNLOADING] sgpceil10mC1.b1.20191224.000016.nc
[DOWNLOADING] sgpceil10mC1.b1.20191225.000014.nc
[DOWNLOADING] sgpceil10mC1.b1.20191226.000011.nc
[DOWNLOADING] sgpceil10mC1.b1.20191227.000009.nc
[DOWNLOADING] sgpceil10mC1.b1.20191228.000006.nc
[DOWNLOADING] sgpceil10mC1.b1.20191229.000004.nc
[DOWNLOADING] sgpceil10mC1.b1.20191230.000002.nc
If you use these data to prepare a publication, please cite:
Zhang, D., Morris, V., & Ermold, B. Ceilometer (CEIL10M), 2019-11-01 to
2019-12-31, Southern Great Plains (SGP), Central Facility, Lamont, OK (C1).
Atmospheric Radiation Measurement (ARM) User Facility.
https://doi.org/10.5439/1497398
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191101.000035.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191102.000923.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191103.001904.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191104.002756.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191105.000335.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191106.000258.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191107.000203.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191108.000117.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191109.001016.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191110.001902.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191111.002834.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191112.000433.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191113.000912.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191114.000830.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191115.001735.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191116.002641.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191117.000247.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191118.000145.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191119.000049.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191120.000005.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191121.000910.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191122.001805.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191123.002721.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191124.000300.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191125.000208.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191126.000125.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191127.000040.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191128.000934.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191129.001840.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191130.002745.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191201.000421.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191202.000258.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191203.000203.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191204.000105.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191205.001023.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191206.001917.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191207.002833.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191208.000449.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191209.000334.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191210.000242.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191211.000158.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191212.001059.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191213.002008.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191214.002908.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191215.000512.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191216.000414.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191217.000320.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191218.000225.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191219.001129.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191220.002048.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191221.002950.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191222.000546.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191223.000452.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191224.000356.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191225.000304.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191226.001207.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191227.002123.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191228.003029.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191229.000644.nc
[DOWNLOADING] sgpaosccn2colaavgE13.b1.20191230.000529.nc
If you use these data to prepare a publication, please cite:
Koontz, A., Uin, J., Andrews, E., Enekwizu, O., Hayes, C., & Salwen, C. Cloud
Condensation Nuclei Particle Counter (AOSCCN2COLAAVG), 2019-11-01 to 2019-12-31,
Southern Great Plains (SGP), Lamont, OK (Extended and Co-located with C1) (E13).
Atmospheric Radiation Measurement (ARM) User Facility.
https://doi.org/10.5439/1323894
df1 = act.io.read_arm_netcdf(result_datastream1_met)
df2 = act.io.read_arm_netcdf(result_datastream2_met)
df3 = act.io.read_arm_netcdf(result_datastream3_met)
df4 = act.io.read_arm_netcdf(result_datastream4_met)
df5 = act.io.read_arm_netcdf(result_datastream5_met)
df6 = act.io.read_arm_netcdf(result_datastream6_met)print(df1.sizes,"\n", df2.sizes,"\n", df3.sizes,"\n", df4.sizes,"\n", df5.sizes,"\n", df6.sizes)Frozen({'time': 1464, 'bound': 2, 'merged_diameter_mobility': 212, 'diameter_aerodynamic': 52, 'diameter_mobility': 192})
Frozen({'time': 6051, 'bound': 2})
Frozen({'time': 86400, 'bound': 2})
Frozen({'time': 5177123})
Frozen({'time': 323546, 'bound': 2, 'range': 770})
Frozen({'time': 5577, 'bound': 2, 'droplet_size': 20, 'setpoint': 6})
print(df1.time,"\n", df2.time,"\n", df3.time,"\n", df4.time,"\n", df5.time,"\n", df5.time)<xarray.DataArray 'time' (time: 1464)> Size: 12kB
array(['2019-11-01T00:00:00.000000000', '2019-11-01T01:00:00.000000000',
'2019-11-01T02:00:00.000000000', ..., '2019-12-31T21:00:00.000000000',
'2019-12-31T22:00:00.000000000', '2019-12-31T23:00:00.000000000'],
shape=(1464,), dtype='datetime64[ns]')
Coordinates:
* time (time) datetime64[ns] 12kB 2019-11-01 ... 2019-12-31T23:00:00
Attributes:
long_name: Time offset from midnight
bounds: time_bounds
standard_name: time
<xarray.DataArray 'time' (time: 6051)> Size: 48kB
array(['2019-11-01T00:07:41.000000000', '2019-11-01T00:17:58.000000000',
'2019-11-01T00:28:16.000000000', ..., '2019-12-30T22:31:16.000000000',
'2019-12-30T23:01:07.000000000', '2019-12-30T23:31:39.000000000'],
shape=(6051,), dtype='datetime64[ns]')
Coordinates:
* time (time) datetime64[ns] 48kB 2019-11-01T00:07:41 ... 2019-12-30T23...
Attributes:
long_name: Time offset from midnight
bounds: time_bounds
standard_name: time
<xarray.DataArray 'time' (time: 86400)> Size: 691kB
array(['2019-11-01T00:00:30.000000000', '2019-11-01T00:01:30.000000000',
'2019-11-01T00:02:30.000000000', ..., '2019-12-30T23:57:30.000000000',
'2019-12-30T23:58:30.000000000', '2019-12-30T23:59:30.000000000'],
shape=(86400,), dtype='datetime64[ns]')
Coordinates:
* time (time) datetime64[ns] 691kB 2019-11-01T00:00:30 ... 2019-12-30T2...
Attributes:
long_name: Time offset from midnight
bounds: time_bounds
standard_name: time
<xarray.DataArray 'time' (time: 5177123)> Size: 41MB
array(['2019-11-01T00:00:00.580000000', '2019-11-01T00:00:01.580000000',
'2019-11-01T00:00:02.580000000', ..., '2019-12-30T23:59:57.570000000',
'2019-12-30T23:59:58.570000000', '2019-12-30T23:59:59.570000000'],
shape=(5177123,), dtype='datetime64[ns]')
Coordinates:
* time (time) datetime64[ns] 41MB 2019-11-01T00:00:00.580000 ... 2019-1...
Attributes:
long_name: Time offset from midnight
standard_name: time
<xarray.DataArray 'time' (time: 323546)> Size: 3MB
array(['2019-11-01T00:00:01.000000000', '2019-11-01T00:00:17.000000000',
'2019-11-01T00:00:33.000000000', ..., '2019-12-30T23:59:27.000000000',
'2019-12-30T23:59:43.000000000', '2019-12-30T23:59:59.000000000'],
shape=(323546,), dtype='datetime64[ns]')
Coordinates:
* time (time) datetime64[ns] 3MB 2019-11-01T00:00:01 ... 2019-12-30T23:...
Attributes:
long_name: Time offset from midnight
bounds: time_bounds
standard_name: time
<xarray.DataArray 'time' (time: 323546)> Size: 3MB
array(['2019-11-01T00:00:01.000000000', '2019-11-01T00:00:17.000000000',
'2019-11-01T00:00:33.000000000', ..., '2019-12-30T23:59:27.000000000',
'2019-12-30T23:59:43.000000000', '2019-12-30T23:59:59.000000000'],
shape=(323546,), dtype='datetime64[ns]')
Coordinates:
* time (time) datetime64[ns] 3MB 2019-11-01T00:00:01 ... 2019-12-30T23:...
Attributes:
long_name: Time offset from midnight
bounds: time_bounds
standard_name: time
print(df1.time.diff("time")[:5],"\n",df2.time.diff("time")[:5],"\n",df3.time.diff("time")[:5],"\n",df4.time.diff("time")[:5],"\n",df5.time.diff("time")[:5],"\n",df6.time.diff("time")[:5])<xarray.DataArray 'time' (time: 5)> Size: 40B
array([3600000000000, 3600000000000, 3600000000000, 3600000000000,
3600000000000], dtype='timedelta64[ns]')
Coordinates:
* time (time) datetime64[ns] 40B 2019-11-01T01:00:00 ... 2019-11-01T05:...
Attributes:
long_name: Time offset from midnight
bounds: time_bounds
standard_name: time
<xarray.DataArray 'time' (time: 5)> Size: 40B
array([617000000000, 618000000000, 617000000000, 617000000000,
618000000000], dtype='timedelta64[ns]')
Coordinates:
* time (time) datetime64[ns] 40B 2019-11-01T00:17:58 ... 2019-11-01T00:...
Attributes:
long_name: Time offset from midnight
bounds: time_bounds
standard_name: time
<xarray.DataArray 'time' (time: 5)> Size: 40B
array([60000000000, 60000000000, 60000000000, 60000000000, 60000000000],
dtype='timedelta64[ns]')
Coordinates:
* time (time) datetime64[ns] 40B 2019-11-01T00:01:30 ... 2019-11-01T00:...
Attributes:
long_name: Time offset from midnight
bounds: time_bounds
standard_name: time
<xarray.DataArray 'time' (time: 5)> Size: 40B
array([1000000000, 1000000000, 1000000000, 1000000000, 1000000000],
dtype='timedelta64[ns]')
Coordinates:
* time (time) datetime64[ns] 40B 2019-11-01T00:00:01.580000 ... 2019-11...
Attributes:
long_name: Time offset from midnight
standard_name: time
<xarray.DataArray 'time' (time: 5)> Size: 40B
array([16000000000, 16000000000, 16000000000, 16000000000, 16000000000],
dtype='timedelta64[ns]')
Coordinates:
* time (time) datetime64[ns] 40B 2019-11-01T00:00:17 ... 2019-11-01T00:...
Attributes:
long_name: Time offset from midnight
bounds: time_bounds
standard_name: time
<xarray.DataArray 'time' (time: 5)> Size: 40B
array([ 592000000000, 598000000000, 619000000000, 1948000000000,
610000000000], dtype='timedelta64[ns]')
Coordinates:
* time (time) datetime64[ns] 40B 2019-11-01T00:10:27 ... 2019-11-01T01:...
Attributes:
long_name: Time offset from midnight
standard_name: time
bounds: time_bounds
print(df1.data_vars,"\n", df2.data_vars,"\n", df3.data_vars,"\n", df4.data_vars,"\n", df5.data_vars,"\n", df6.data_vars)Data variables:
base_time (time) datetime64[ns] 12kB 2019-11-0...
time_offset (time) datetime64[ns] 12kB 2019-11-0...
time_bounds (time, bound) object 23kB dask.array<chunksize=(24, 2), meta=np.ndarray>
merged_diameter_mobility_bounds (time, merged_diameter_mobility, bound) float64 5MB dask.array<chunksize=(24, 212, 2), meta=np.ndarray>
diameter_aerodynamic_bounds (time, diameter_aerodynamic, bound) float32 609kB dask.array<chunksize=(24, 52, 2), meta=np.ndarray>
diameter_mobility_bounds (time, diameter_mobility, bound) float32 2MB dask.array<chunksize=(24, 192, 2), meta=np.ndarray>
effective_density (time) float32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
qc_effective_density (time) int32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
effective_density_solution_strength (time) float64 12kB dask.array<chunksize=(24,), meta=np.ndarray>
merged_dN_dlogDp (time, merged_diameter_mobility) float32 1MB dask.array<chunksize=(24, 212), meta=np.ndarray>
qc_merged_dN_dlogDp (time, merged_diameter_mobility) int32 1MB dask.array<chunksize=(24, 212), meta=np.ndarray>
merged_total_N_conc (time) float32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
qc_merged_total_N_conc (time) int32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
merged_total_SA_conc (time) float32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
qc_merged_total_SA_conc (time) int32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
merged_total_V_conc (time) float32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
qc_merged_total_V_conc (time) int32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
aps_total_N_conc (time) float32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
smps_total_N_conc (time) float32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
aps_dN_dlogDp (time, diameter_aerodynamic) float32 305kB dask.array<chunksize=(24, 52), meta=np.ndarray>
smps_dN_dlogDp (time, diameter_mobility) float32 1MB dask.array<chunksize=(24, 192), meta=np.ndarray>
resid_DA_num (time) float32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
resid_DA_vol (time) float32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
resid_DE_num (time) float32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
resid_DE_vol (time) float32 6kB dask.array<chunksize=(24,), meta=np.ndarray>
lat (time) float32 6kB 36.6 36.6 ... 36.6
lon (time) float32 6kB -97.49 ... -97.49
alt (time) float32 6kB 318.0 ... 318.0
Data variables:
base_time (time) datetime64[ns] 48kB 2019-11-01 ... 2...
time_offset (time) datetime64[ns] 48kB 2019-11-01T00:07...
time_bounds (time, bound) object 97kB dask.array<chunksize=(140, 2), meta=np.ndarray>
total_organics_CDCE (time) float32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
qc_total_organics_CDCE (time) int32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
sulfate_CDCE (time) float32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
qc_sulfate_CDCE (time) int32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
ammonium_CDCE (time) float32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
qc_ammonium_CDCE (time) int32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
nitrate_CDCE (time) float32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
qc_nitrate_CDCE (time) int32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
chloride_CDCE (time) float32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
qc_chloride_CDCE (time) int32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
acsm_vol_conc (time) float32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
ammonium_predicted (time) float32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
CDCE (time) float32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
airbeam_normalization_factor (time) float32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
inlet_pressure (time) float32 24kB dask.array<chunksize=(140,), meta=np.ndarray>
lat (time) float32 24kB 36.6 36.6 ... 36.6 36.6
lon (time) float32 24kB -97.49 -97.49 ... -97.49
alt (time) float32 24kB 318.0 318.0 ... 318.0
Data variables:
base_time (time) datetime64[ns] 691kB 2019-11-01 .....
time_offset (time) datetime64[ns] 691kB 2019-11-01T00...
time_bounds (time, bound) object 1MB dask.array<chunksize=(1440, 2), meta=np.ndarray>
impactor_state (time) float64 691kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_B (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bs_B (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_G (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bs_G (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_R (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bs_R (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AE_BG (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_AE_BG (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AE_BR (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_AE_BR (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AE_GR (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_AE_GR (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bbs_B (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bbs_B (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bbs_G (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bbs_G (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bbs_R (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bbs_R (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AE_Bbs_BG (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_AE_Bbs_BG (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AE_Bbs_BR (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_AE_Bbs_BR (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AE_Bbs_GR (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_AE_Bbs_GR (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
bsf_B (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_bsf_B (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
bsf_G (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_bsf_G (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
bsf_R (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_bsf_R (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
g_B (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_g_B (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
g_G (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_g_G (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
g_R (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_g_R (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_B_combined (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_B_combined (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_G_combined (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_G_combined (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_R_combined (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_R_combined (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AAE_BG (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_AAE_BG (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AAE_BR (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_AAE_BR (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AAE_GR (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_AAE_GR (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
ssa_B (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_ssa_B (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
ssa_G (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_ssa_G (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
ssa_R (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_ssa_R (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_B_Virkkula (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_B_Virkkula (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_G_Virkkula (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_G_Virkkula (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_R_Virkkula (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_R_Virkkula (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
ssa_B_Virkkula (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_ssa_B_Virkkula (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
ssa_G_Virkkula (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_ssa_G_Virkkula (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
ssa_R_Virkkula (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_ssa_R_Virkkula (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_B_raw (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_B_raw (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_G_raw (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_G_raw (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_R_raw (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_R_raw (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
sample_flow_rate (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_sample_flow_rate (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
transmittance_blue (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_transmittance_blue (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
transmittance_green (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_transmittance_green (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
transmittance_red (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_transmittance_red (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_B_BondOgren (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_B_BondOgren (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_G_BondOgren (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_G_BondOgren (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_R_BondOgren (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_R_BondOgren (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_B_Dry_Neph3W_uncorrected (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bs_B_Dry_Neph3W_uncorrected (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_G_Dry_Neph3W_uncorrected (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bs_G_Dry_Neph3W_uncorrected (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_R_Dry_Neph3W_uncorrected (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bs_R_Dry_Neph3W_uncorrected (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
P_Neph_Dry (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_P_Neph_Dry (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
T_Neph_Dry (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_T_Neph_Dry (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
RH_Neph_Dry (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_RH_Neph_Dry (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AE_Bs_BG_uncorrected (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AE_Bs_BR_uncorrected (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
AE_Bs_GR_uncorrected (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_B_uncorrected (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_G_uncorrected (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_R_uncorrected (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_B_Dry_Neph3W (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bs_B_Dry_Neph3W (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_G_Dry_Neph3W (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bs_G_Dry_Neph3W (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bs_R_Dry_Neph3W (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bs_R_Dry_Neph3W (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bbs_B_Dry_Neph3W (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bbs_B_Dry_Neph3W (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bbs_G_Dry_Neph3W (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bbs_G_Dry_Neph3W (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Bbs_R_Dry_Neph3W (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Bbs_R_Dry_Neph3W (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_B_Weiss (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_B_Weiss (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_G_Weiss (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_G_Weiss (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
Ba_R_Weiss (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
qc_Ba_R_Weiss (time) int32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
K1_B (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
K1_G (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
K1_R (time) float32 346kB dask.array<chunksize=(1440,), meta=np.ndarray>
lat (time) float32 346kB 36.6 36.6 ... 36.6 36.6
lon (time) float32 346kB -97.49 ... -97.49
alt (time) float32 346kB 318.0 318.0 ... 318.0
Data variables:
base_time (time) datetime64[ns] 41MB 2019-11-01 ... 2019-12-30
time_offset (time) datetime64[ns] 41MB 2019-11-01T00:00:00.58000...
rh_ambient (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
temperature_ambient (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
pressure_ambient (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
wind_speed (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
wind_direction (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
rain_amount (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
rain_duration (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
rain_intensity (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
heater_temp (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
heater_volts (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
supply_volts (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
ref_volts (time) float32 21MB dask.array<chunksize=(86400,), meta=np.ndarray>
lat (time) float32 21MB 36.61 36.61 36.61 ... 36.61 36.61
lon (time) float32 21MB -97.49 -97.49 ... -97.49 -97.49
alt (time) float32 21MB 316.2 316.2 316.2 ... 316.2 316.2
Data variables:
base_time (time) datetime64[ns] 3MB 2019-11-01 ... 2019-12-30
time_offset (time) datetime64[ns] 3MB 2019-11-01T00:00:01 ......
time_bounds (time, bound) object 5MB dask.array<chunksize=(5401, 2), meta=np.ndarray>
range_bounds (time, range, bound) float32 2GB dask.array<chunksize=(5401, 770, 2), meta=np.ndarray>
detection_status (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
status_flag (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
first_cbh (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
qc_first_cbh (time) int32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
vertical_visibility (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
qc_vertical_visibility (time) int32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
second_cbh (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
qc_second_cbh (time) int32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
alt_highest_signal (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
qc_alt_highest_signal (time) int32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
third_cbh (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
qc_third_cbh (time) int32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
laser_pulse_energy (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
qc_laser_pulse_energy (time) int32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
laser_temperature (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
qc_laser_temperature (time) int32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
window_transmission (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
qc_window_transmission (time) int32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
tilt_angle (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
qc_tilt_angle (time) int32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
background_light (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
qc_background_light (time) int32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
sum_backscatter (time) float32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
qc_sum_backscatter (time) int32 1MB dask.array<chunksize=(5401,), meta=np.ndarray>
backscatter (time, range) float32 997MB dask.array<chunksize=(5401, 770), meta=np.ndarray>
measurement_parameters (time) |S12 4MB dask.array<chunksize=(5401,), meta=np.ndarray>
status_string (time) |S12 4MB dask.array<chunksize=(5401,), meta=np.ndarray>
lat (time) float32 1MB 36.6 36.6 36.6 ... 36.6 36.6 36.6
lon (time) float32 1MB -97.49 -97.49 ... -97.49 -97.49
alt (time) float32 1MB 318.0 318.0 318.0 ... 318.0 318.0
Data variables:
base_time (time) datetime64[ns] 45kB 2019-11-01 ... 2...
time_offset (time) datetime64[ns] 45kB 2019-11-01T00:00...
time_bounds (time, bound) object 89kB dask.array<chunksize=(93, 2), meta=np.ndarray>
droplet_size_bounds (time, droplet_size, bound) float32 892kB dask.array<chunksize=(93, 20, 2), meta=np.ndarray>
Q_sample (time) float32 22kB dask.array<chunksize=(93,), meta=np.ndarray>
overflow (time) float32 22kB dask.array<chunksize=(93,), meta=np.ndarray>
supersaturation_calculated (time) float32 22kB dask.array<chunksize=(93,), meta=np.ndarray>
aerosol_number_concentration (time) float32 22kB dask.array<chunksize=(93,), meta=np.ndarray>
N_CCN (time) float32 22kB dask.array<chunksize=(93,), meta=np.ndarray>
qc_N_CCN (time) int32 22kB dask.array<chunksize=(93,), meta=np.ndarray>
N_CCN_dN (time, droplet_size) float32 446kB dask.array<chunksize=(93, 20), meta=np.ndarray>
first_bin_used (time) float64 45kB dask.array<chunksize=(93,), meta=np.ndarray>
lat (time) float32 22kB 36.61 36.61 ... 36.61
lon (time) float32 22kB -97.49 -97.49 ... -97.49
alt (time) float32 22kB 316.2 316.2 ... 316.2
df6["supersaturation_calculated"].to_series().describe()count 5577.000000
mean 0.382399
std 0.283492
min 0.086439
25% 0.189530
50% 0.385248
75% 0.792618
max 1.034438
Name: supersaturation_calculated, dtype: float64Preprocessing¶
"""
Prepare machine-learning dataframe for CCN prediction.
"""
# =============================================================================
# Select variables
# =============================================================================
df1_dist = df1[["merged_dN_dlogDp"]]
df2_vars = df2[["nitrate_CDCE", "sulfate_CDCE", "ammonium_CDCE"]]
df3_vars = df3[["Ba_G_combined", "Bs_G", "AAE_BR", "AE_BR"]]
df4_vars = df4[["rh_ambient", "temperature_ambient"]]
df6_vars = df6[["N_CCN"]]
# =============================================================================
# Apply QC filtering
# =============================================================================
df1_dist["merged_dN_dlogDp"] = df1_dist["merged_dN_dlogDp"].where(df1["qc_merged_dN_dlogDp"] == 0)
df2_vars["nitrate_CDCE"] = df2_vars["nitrate_CDCE"].where(df2["qc_nitrate_CDCE"] == 0)
df2_vars["sulfate_CDCE"] = df2_vars["sulfate_CDCE"].where(df2["qc_sulfate_CDCE"] == 0)
df2_vars["ammonium_CDCE"] = df2_vars["ammonium_CDCE"].where(df2["qc_ammonium_CDCE"] == 0)
df3_vars["Bs_G"] = df3_vars["Bs_G"].where(df3["qc_Bs_G"] == 0)
df3_vars["Ba_G_combined"] = df3_vars["Ba_G_combined"].where(df3["qc_Ba_G_combined"] == 0)
df3_vars["AAE_BR"] = df3_vars["AAE_BR"].where(df3["qc_AAE_BR"] == 0)
df3_vars["AE_BR"] = df3_vars["AE_BR"].where(df3["qc_AE_BR"] == 0)
df6_vars["N_CCN"] = df6_vars["N_CCN"].where(df6["qc_N_CCN"] == 0)
# =============================================================================
# Reduce aerosol size distribution
# =============================================================================
diam = df1["merged_diameter_mobility"]
dN_small = df1_dist["merged_dN_dlogDp"][:, diam < 80].sum(dim="merged_diameter_mobility")
dN_medium = df1_dist["merged_dN_dlogDp"][:, (diam >= 80) & (diam < 200)].sum(dim="merged_diameter_mobility")
dN_large = df1_dist["merged_dN_dlogDp"][:, diam >= 200].sum(dim="merged_diameter_mobility")
df1_reduced = xr.Dataset({"dN_small": dN_small, "dN_medium": dN_medium, "dN_large": dN_large})
# =============================================================================
# Add bulk aerosol variables
# =============================================================================
df1_bulk = df1[["merged_total_SA_conc", "merged_total_V_conc", "effective_density"]]
df1_bulk["merged_total_SA_conc"] = df1_bulk["merged_total_SA_conc"].where(df1["qc_merged_total_SA_conc"] == 0)
df1_bulk["merged_total_V_conc"] = df1_bulk["merged_total_V_conc"].where(df1["qc_merged_total_V_conc"] == 0)
df1_bulk["effective_density"] = df1_bulk["effective_density"].where(df1["qc_effective_density"] == 0)
# =============================================================================
# Final aerosol dataset
# =============================================================================
df1_vars = xr.merge([df1_reduced, df1_bulk])
# =============================================================================
# Resample to common 10-minute resolution
# =============================================================================
df1_10m = df1_vars.resample(time="10min").nearest()
df2_10m = df2_vars.resample(time="10min").nearest()
df3_10m = df3_vars.resample(time="10min").mean()
df4_10m = df4_vars.resample(time="10min").mean()
df6_10m = df6_vars.resample(time="10min").mean()
# =============================================================================
# Merge datasets
# =============================================================================
combined = xr.merge([df1_10m, df2_10m, df3_10m, df4_10m, df6_10m])
# =============================================================================
# Convert to dataframe
# =============================================================================
df_ml = combined.to_dataframe().reset_index()
# =============================================================================
# Check missing values
# =============================================================================
print("\nMissing fraction:\n")
print(df_ml.isnull().mean())
# =============================================================================
# Remove missing values
# =============================================================================
df_ml = df_ml.dropna()
# =============================================================================
# Remove unnecessary columns
# =============================================================================
drop_cols = ["lat", "lon", "alt", "bound"]
for col in drop_cols:
if col in df_ml.columns:
df_ml = df_ml.drop(columns=col)
# =============================================================================
# Final dataframe check
# =============================================================================
print("\nFinal dataframe shape:\n")
print(df_ml.shape)
print("\nFirst few rows:\n")
print(df_ml.head())/tmp/ipykernel_464/3452497034.py:61: FutureWarning: In a future version of xarray the default value for join will change from join='outer' to join='exact'. This change will result in the following ValueError: cannot be aligned with join='exact' because index/labels/sizes are not equal along these coordinates (dimensions): 'time' ('time',) The recommendation is to set join explicitly for this case.
combined = xr.merge([df1_10m, df2_10m, df3_10m, df4_10m, df6_10m])
Missing fraction:
time 0.000000
dN_small 0.000000
dN_medium 0.000000
dN_large 0.000000
merged_total_SA_conc 0.082697
merged_total_V_conc 0.082697
effective_density 0.069028
nitrate_CDCE 0.046133
sulfate_CDCE 0.045791
ammonium_CDCE 0.065497
Ba_G_combined 0.455974
Bs_G 0.098872
AAE_BR 0.516688
AE_BR 0.098758
rh_ambient 0.016631
temperature_ambient 0.016631
N_CCN 0.370202
dtype: float64
Final dataframe shape:
(2323, 17)
First few rows:
time dN_small dN_medium dN_large \
674 2019-11-05 16:20:00 215180.828125 25791.052734 5694.370605
675 2019-11-05 16:30:00 398837.843750 24004.023438 4234.035156
676 2019-11-05 16:40:00 398837.843750 24004.023438 4234.035156
677 2019-11-05 16:50:00 398837.843750 24004.023438 4234.035156
680 2019-11-05 17:20:00 398837.843750 24004.023438 4234.035156
merged_total_SA_conc merged_total_V_conc effective_density \
674 4.040957e+09 5.230564e+11 1.6
675 3.916287e+09 5.006706e+11 1.5
676 3.916287e+09 5.006706e+11 1.5
677 3.916287e+09 5.006706e+11 1.5
680 3.916287e+09 5.006706e+11 1.5
nitrate_CDCE sulfate_CDCE ammonium_CDCE Ba_G_combined Bs_G \
674 0.369653 0.592129 0.207426 0.500800 9.278229
675 0.353300 0.274260 0.260589 0.366442 8.599729
676 0.254471 0.470293 0.375469 0.206045 8.297157
677 0.211970 0.256478 0.232902 0.146883 7.404489
680 0.152774 0.420142 0.188616 0.058656 5.032202
AAE_BR AE_BR rh_ambient temperature_ambient N_CCN
674 2.353307 2.371831 52.343002 10.611835 134.346008
675 2.378701 2.375671 49.989002 11.120833 476.214752
676 2.492949 2.348520 48.654335 11.520666 1308.750977
677 2.131954 2.839947 47.477169 11.758333 2069.727051
680 0.981215 2.944649 44.077671 12.587500 100.546883
df_ml.to_csv("/data/home/abhigyan/total_smokeshow/data/data.csv")EDA and Machine Learning¶
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import xgboost as xgb
import shap
from sklearn.metrics import r2_score, mean_squared_error
from sklearn.inspection import PartialDependenceDisplay, permutation_importancedf_ml = pd.read_csv("/data/home/user/total_smokeshow/data/data.csv")df_ml.columnsIndex(['Unnamed: 0', 'time', 'dN_small', 'dN_medium', 'dN_large',
'merged_total_SA_conc', 'merged_total_V_conc', 'effective_density',
'nitrate_CDCE', 'sulfate_CDCE', 'ammonium_CDCE', 'Ba_G_combined',
'Bs_G', 'AAE_BR', 'AE_BR', 'rh_ambient', 'temperature_ambient',
'N_CCN'],
dtype='str')# --- Global Visual Settings ---
plt.rcParams['font.family'] = 'serif'
PRIMARY_COLOR = '#FFCC99' # Pastel Orange
SECONDARY_COLOR = '#333333' # Professional Black/Grey
"""
SECTION 2: EXPLORATORY DATA ANALYSIS (EDA)
Visualizing distributions and correlations to understand data quality.
"""
# 2.1 Histograms of all variables
df_hist = df_ml.drop(columns=['Unnamed: 0','N_CCN', 'time'])
df_hist.hist(bins=20, figsize=(12, 10), color=PRIMARY_COLOR, edgecolor=SECONDARY_COLOR)
plt.suptitle('Variable Distributions', fontsize=16)
plt.tight_layout()
plt.savefig("/data/home/user/total_smokeshow/Results/Histogram.png")
plt.show()
# 2.2 Pearson Correlation Heatmap
plt.figure(figsize=(10, 8))
df_corr = df_ml.drop(columns=['Unnamed: 0','N_CCN', 'time'])
corr = df_corr.corr()
sns.heatmap(corr, annot=True, vmin=-1, vmax=1, cmap='RdBu_r', fmt=".2f", cbar=True)
plt.title('Feature Correlation Heatmap (Pearson)', fontsize=14)
plt.tight_layout(rect=[0, 0, 0.9, 0.9])
plt.savefig("/data/home/user/total_smokeshow/Results/Correlation.png")
plt.show()
# 2.3 Baseline XGBoost Importance
# Quick baseline feature importance check
X_base = df_ml.drop(['Unnamed: 0', 'N_CCN', 'time'], axis=1)
y_base = df_ml['N_CCN']
temp_model = xgb.XGBRegressor(
random_state=41
)
temp_model.fit(X_base, y_base)
plt.figure(figsize=(8, 5))
xgb.plot_importance(
temp_model,
color=PRIMARY_COLOR,
grid=False,
importance_type='weight'
)
plt.title('Baseline XGBoost Feature Importance')
plt.tight_layout(rect=[0, 0, 1, 0.9])
plt.savefig(
"/data/home/user/total_smokeshow/Results/Feature_importance.png",
dpi=300,
bbox_inches='tight'
)
plt.show()
<Figure size 800x500 with 0 Axes>
"""
SECTION 3: ML MODELLING
Handling the time-series split, hyperparameter tuning, training,
and uncertainty analysis.
"""
from sklearn.model_selection import GridSearchCV, TimeSeriesSplit
from sklearn.metrics import r2_score, mean_squared_error
import xgboost as xgb
import numpy as np
import matplotlib.pyplot as plt
import scipy.stats as stats
# =============================================================================
# 3.0 Data Preparation
# =============================================================================
df_mask = df_ml.drop(
[
'Unnamed: 0',
'time',
"merged_total_SA_conc",
"merged_total_V_conc",
"dN_large",
# "Bs_G",
# "ammonium_CDCE",
"effective_density"
],
axis=1
)
# =============================================================================
# 3.1 Time-Sequential Split (80% Train, 20% Test)
# =============================================================================
split_idx = int(len(df_mask) * 0.8)
train_df = df_mask.iloc[:split_idx]
test_df = df_mask.iloc[split_idx:]
X_train = train_df.drop('N_CCN', axis=1)
y_train = train_df['N_CCN']
X_test = test_df.drop('N_CCN', axis=1)
y_test = test_df['N_CCN']
# =============================================================================
# 3.2 Hyperparameter Tuning with TimeSeriesSplit
# =============================================================================
xgb_model = xgb.XGBRegressor(
objective='reg:squarederror',
random_state=42
)
param_grid = {
'n_estimators': [100, 200],
'learning_rate': [0.03, 0.05, 0.1],
'max_depth': [3, 4, 5],
'subsample': [0.7, 0.8, 1.0],
'colsample_bytree': [0.7, 0.8, 1.0]
}
tscv = TimeSeriesSplit(n_splits=5)
grid_search = GridSearchCV(
estimator=xgb_model,
param_grid=param_grid,
cv=tscv,
scoring='r2',
n_jobs=-1,
verbose=1
)
grid_search.fit(X_train, y_train)
# =============================================================================
# 3.3 Best Model
# =============================================================================
model = grid_search.best_estimator_
print("\nBest Parameters:")
print(grid_search.best_params_)
# =============================================================================
# 3.4 Performance Metrics
# =============================================================================
preds = model.predict(X_test)
r2 = r2_score(y_test, preds)
rmse = np.sqrt(mean_squared_error(y_test, preds))
print(f"\nModel Performance:")
print(f"R2 Score: {r2:.3f}")
print(f"RMSE: {rmse:.3f}")
# =============================================================================
# 3.5 Uncertainty Analysis
# =============================================================================
residuals = y_test - preds
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))
# Residual Plot
ax1.scatter(
y_test,
residuals,
color=PRIMARY_COLOR,
alpha=0.6,
edgecolors=SECONDARY_COLOR
)
ax1.axhline(0, color=SECONDARY_COLOR, linestyle='--')
ax1.set_xlabel('Measured CCN')
ax1.set_ylabel('Residual (Error)')
ax1.set_title('Residual Analysis: Error vs. Measured CCN')
ax1.grid(True, linestyle='--', alpha=0.3)
# Q-Q Plot
stats.probplot(residuals, dist="norm", plot=ax2)
ax2.get_lines()[0].set_markerfacecolor(PRIMARY_COLOR)
ax2.get_lines()[0].set_markeredgecolor(SECONDARY_COLOR)
ax2.get_lines()[0].set_alpha(0.6)
ax2.get_lines()[1].set_color(SECONDARY_COLOR)
ax2.set_title('Uncertainty Analysis: Residual Q-Q Plot')
ax2.set_xlabel('Theoretical Quantiles')
ax2.set_ylabel('Ordered Residuals')
ax2.grid(True, linestyle='--', alpha=0.3)
plt.tight_layout()
plt.savefig(
"/data/home/user/total_smokeshow/Results/Residual.png"
)
plt.show()Fitting 5 folds for each of 162 candidates, totalling 810 fits
Best Parameters:
{'colsample_bytree': 1.0, 'learning_rate': 0.03, 'max_depth': 3, 'n_estimators': 100, 'subsample': 0.7}
Model Performance:
R2 Score: 0.210
RMSE: 639.965
"""
SECTION 4: EXPLAINABLE AI (XAI)
Interpreting the model using SHAP, Permutation Importance, and PDPs.
"""
# 4.1 Permutation Feature Importance
perm_importance = permutation_importance(model, X_test, y_test, n_repeats=10, random_state=42)
sorted_idx = perm_importance.importances_mean.argsort()
plt.figure(figsize=(10, 6)) # wider figure
plt.barh(X_test.columns[sorted_idx], perm_importance.importances_mean[sorted_idx], color=PRIMARY_COLOR)
plt.title("Permutation Feature Importance (Test Set)")
plt.tight_layout() # auto-adjusts margins to fit labels
plt.savefig("/data/home/user/total_smokeshow/Results/PFI.png", bbox_inches='tight') # key fix
plt.show()
# 4.2 Partial Dependence Plots (PDP)
top_features = X_test.columns[sorted_idx][:]
fig, ax = plt.subplots(figsize=(14, 10)) # larger figure
PartialDependenceDisplay.from_estimator(model, X_test, top_features, ax=ax)
plt.suptitle('Partial Dependence Plots', fontsize=14)
plt.tight_layout(rect=[0, 0, 1, 0.9])
plt.savefig(
"/data/home/user/total_smokeshow/Results/PDP.png",
bbox_inches='tight', # key fix
pad_inches=0.3 # adds padding around the figure
)
plt.show()
# 4.3 SHAP Values
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(X_test)
# SHAP Beeswarm
plt.figure(figsize=(10, 6))
shap.summary_plot(shap_values, X_test, show=False)
plt.title('SHAP Summary (Beeswarm)')
plt.savefig("/data/home/user/total_smokeshow/Results/SHAP_beeswarm.png")
plt.show()
# SHAP Dependence Plot (Example: Supersaturation)
# This shows how a specific feature interacts with others
features = ['dN_small', 'dN_medium', 'nitrate_CDCE', 'sulfate_CDCE', 'ammonium_CDCE', 'Ba_G_combined', 'Bs_G', 'AAE_BR', 'AE_BR', 'rh_ambient', 'temperature_ambient']
for feature in features:
plt.figure(figsize=(8, 6))
shap.dependence_plot(feature, shap_values, X_test, interaction_index='auto', show=False)
plt.title(f'SHAP Dependence: {feature}')
plt.savefig(f"/data/home/user/total_smokeshow/Results/SHAP_dependence_{feature}.png")
plt.show()
<Figure size 800x600 with 0 Axes>
<Figure size 800x600 with 0 Axes>
<Figure size 800x600 with 0 Axes>
<Figure size 800x600 with 0 Axes>
<Figure size 800x600 with 0 Axes>
<Figure size 800x600 with 0 Axes>
<Figure size 800x600 with 0 Axes>
<Figure size 800x600 with 0 Axes>
<Figure size 800x600 with 0 Axes>
<Figure size 800x600 with 0 Axes>
<Figure size 800x600 with 0 Axes>