
Condensed Particle Counter (CPC) Investigation¶
Load in necessary libraries¶
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import os
from datetime import datetime, timedelta
import fnmatch
import matplotlib.dates as mdates
import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm
import pytz
import math
import time
from matplotlib.colors import LinearSegmentedColormap
from matplotlib.colors import ListedColormap, BoundaryNorm
import matplotlib.pyplot as plt
import xarray
import xarray as xr
import numpy as np
import glob
import getpass
import act
# Scikit-learn imports
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
# Dask imports
from distributed import Client, LocalCluster, progress
Download data from ARM directly using workflow below¶
# Note - recommended ARM Live token to be set as an environmental variable
# Check for env variables, otherwise set your username and token here!
if "ARM_USERNAME" in os.environ:
arm_username = os.getenv("ARM_USERNAME")
else:
arm_username = input("Enter ARM username: ").strip()
if "ARM_TOKEN" in os.environ:
arm_token = os.getenv("ARM_TOKEN")
else:
arm_token = getpass.getpass("Enter ARM token (hidden): ").strip()
# Set the datastream and start/enddates
#datastream = 'sgpaoscpcufE13.b1'
datastream = 'sgpaoscpcfE13.b1'
startdate = '2022-12-30'
enddate = '2024-01-02'
# Use ACT to easily download the data. Watch for the data citation! Show some support
# for ARM's instrument experts and cite their data if you use it in a publication
arm_files = act.discovery.download_arm_data(arm_username, arm_token, datastream, startdate, enddate)Enter ARM username: gcarrillocardenas
Enter ARM token (hidden): ········
[DOWNLOADING] sgpaoscpcfE13.b1.20221230.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20221231.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230101.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230102.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230103.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230104.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230105.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230106.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230107.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230108.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230109.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230110.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230111.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230112.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230113.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230114.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230115.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230116.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230117.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230118.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230119.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230120.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230121.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230122.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230123.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230124.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230125.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230126.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230127.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230128.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230129.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230130.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230131.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230201.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230202.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230203.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230204.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230205.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230206.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230207.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230208.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230209.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230210.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230211.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230212.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230213.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230214.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230215.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230216.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230217.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230218.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230219.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230220.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230221.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230222.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230223.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230224.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230225.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230226.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230227.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230228.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230301.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230302.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230303.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230304.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230305.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230306.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230307.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230308.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230309.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230310.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230311.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230312.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230313.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230314.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230315.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230316.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230317.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230318.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230319.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230320.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230321.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230322.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230323.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230324.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230325.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230326.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230327.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230328.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230329.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230330.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230331.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230401.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230402.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230403.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230404.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230405.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230406.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230407.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230408.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230409.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230410.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230411.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230412.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230413.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230414.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230415.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230416.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230417.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230418.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230419.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230420.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230421.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230422.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230423.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230424.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230425.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230426.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230427.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230428.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230429.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230430.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230501.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230502.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230503.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230504.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230505.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230506.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230507.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230508.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230509.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230510.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230511.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230512.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230513.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230514.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230515.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230516.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230517.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230518.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230519.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230520.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230521.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230522.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230523.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230524.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230525.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230526.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230527.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230528.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230529.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230530.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230531.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230601.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230602.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230603.000002.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230604.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230605.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230606.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230607.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230608.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230609.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230610.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230611.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230612.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230613.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230614.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230615.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230616.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230617.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230618.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230619.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230620.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230621.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230622.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230623.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230624.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230625.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230626.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230627.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230628.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230629.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230630.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230701.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230702.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230703.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230704.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230705.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230706.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230707.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230708.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230709.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230710.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230711.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230712.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230713.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230714.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230715.000001.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230716.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230717.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230718.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230719.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230720.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230721.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230722.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230723.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230724.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230725.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230726.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230727.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230728.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230729.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230730.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230731.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230801.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230802.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230803.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230804.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230805.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230806.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230807.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230808.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230809.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230810.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230811.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230812.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230813.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230814.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230815.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230816.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230817.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230818.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230819.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230820.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230821.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230822.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230823.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230824.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230825.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230826.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230827.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230828.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230829.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230830.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230831.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230901.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230902.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230903.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230904.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230905.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230906.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230907.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230908.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230909.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230910.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230911.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230912.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230913.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230914.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230915.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230916.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230917.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230918.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230919.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230920.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230921.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230922.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230923.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230924.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230925.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230926.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230927.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230928.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230929.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20230930.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231001.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231002.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231003.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231004.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231005.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231006.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231007.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231008.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231009.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231010.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231011.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231012.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231013.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231014.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231015.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231016.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231017.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231018.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231019.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231020.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231021.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231022.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231023.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231024.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231025.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231026.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231027.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231028.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231029.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231030.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231031.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231101.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231102.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231103.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231104.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231105.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231106.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231107.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231108.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231109.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231110.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231111.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231112.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231113.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231114.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231115.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231116.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231117.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231118.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231119.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231120.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231121.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231122.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231123.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231124.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231125.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231126.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231127.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231128.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231129.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231130.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231201.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231202.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231203.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231204.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231205.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231206.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231207.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231208.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231209.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231210.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231211.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231212.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231213.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231214.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231215.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231216.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231217.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231218.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231219.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231220.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231221.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231222.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231223.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231224.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231225.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231226.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231227.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231228.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231229.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231230.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20231231.000000.nc
[DOWNLOADING] sgpaoscpcfE13.b1.20240101.000000.nc
If you use these data to prepare a publication, please cite:
Koontz, A., Kuang, C., Andrews, E., Hayes, C., Singh, A., & Salwen, C.
Condensation Particle Counter (AOSCPCF), 2022-12-30 to 2024-01-02, Southern
Great Plains (SGP), Lamont, OK (Extended and Co-located with C1) (E13).
Atmospheric Radiation Measurement (ARM) User Facility.
https://doi.org/10.5439/1352536
path = os.path.expanduser(
'~/DOE_ARM_SummerSchool/sgpaoscpcufE13.b1/sgpaoscpcufE13.b1*.nc'
)
cpcuf_data_files = sorted(glob.glob(path))
len(cpcuf_data_files)
368first_file = cpcuf_data_files[0]
ds = xr.open_dataset(first_file)
variables = list(ds.variables)
print(variables)
['base_time', 'time_offset', 'time', 'concentration', 'qc_concentration', 'saturation_temp', 'condensation_temp', 'optics_temp', 'cabinet_temp', 'ambient_pressure', 'orifice_pressure', 'nozzle_pressure', 'laser_current', 'liquid_level', 'fraction_full', 'corrected_flow', 'lat', 'lon', 'alt']
variable_time = ds["time"]
print(variable_time)
print("Name:", variable_time.name)
print("Dimensions:", variable_time.dims)
print("Shape:", variable_time.shape)
print("Data type:", variable_time.dtype)
print("\nAttributes:")
<xarray.DataArray 'time' (time: 86399)>
array(['2022-12-30T00:00:00.950000000', '2022-12-30T00:00:01.950000000',
'2022-12-30T00:00:02.940000000', ..., '2022-12-30T23:59:57.810000000',
'2022-12-30T23:59:58.810000000', '2022-12-30T23:59:59.810000000'],
dtype='datetime64[ns]')
Coordinates:
* time (time) datetime64[ns] 2022-12-30T00:00:00.950000 ... 2022-12-30T...
Attributes:
long_name: Time offset from midnight
standard_name: time
Name: time
Dimensions: ('time',)
Shape: (86399,)
Data type: datetime64[ns]
Attributes:
variable_concentration = ds["concentration"]
print(variable_concentration)
print("Name:", variable_concentration.name)
print("Dimensions:", variable_concentration.dims)
print("Shape:", variable_concentration.shape)
print("Data type:", variable_concentration.dtype)
print("\nAttributes:")
<xarray.DataArray 'concentration' (time: 86399)>
array([2722.374 , 2703.4688, 2779.0903, ..., 2476.6042, 2542.773 , 2476.6042],
dtype=float32)
Coordinates:
* time (time) datetime64[ns] 2022-12-30T00:00:00.950000 ... 2022-12-30T...
Attributes:
long_name: Particle concentration
units: 1/cm^3
valid_min: 5.0
valid_max: 50000.0
corrections_applied: flow_calibration
flow_calibration_definition: The concentration value has been multiplied...
ancillary_variables: qc_concentration
Name: concentration
Dimensions: ('time',)
Shape: (86399,)
Data type: float32
Attributes:
cpcuf_ds = xr.open_mfdataset(
cpcuf_data_files,
combine="by_coords"
)
# Keep concentration only where qc_concentration == 0
# Usually qc == 0 means good data
# Keep original concentration
# Add QC-fixed concentration as a new xarray variable
cpcuf_ds["concentration_qc_fixed"] = cpcuf_ds["concentration"].where(
cpcuf_ds["qc_concentration"] == 0
)
# Optional: keep only the variables you care about
cpcuf_ds = cpcuf_ds[
["concentration", "qc_concentration", "concentration_qc_fixed"]
]
print(cpcuf_ds)<xarray.Dataset>
Dimensions: (time: 31768745)
Coordinates:
* time (time) datetime64[ns] 2022-12-30T00:00:00.950000 ...
Data variables:
concentration (time) float32 dask.array<chunksize=(86399,), meta=np.ndarray>
qc_concentration (time) int32 dask.array<chunksize=(86399,), meta=np.ndarray>
concentration_qc_fixed (time) float32 dask.array<chunksize=(86399,), meta=np.ndarray>
Attributes: (12/21)
command_line: aoscpcufcorr -s sgp -f E13
Conventions: ARM-1.2
process_version: ingest-aoscpcufcorr-1.5-0.el7
dod_version: aoscpcuf-b1-4.0
site_id: sgp
platform_id: aoscpcuf
... ...
mentor_affiliation: BNL
input_datastreams: sgpaoscpcufE13.a1 : 1.2 : 20221230.000000
configuration_file: sgpaoscpcufE13.20190328.160000.config
AOS_DAC_type: D
doi: 10.5439/1378933
history: created by user dsmgr on machine prod-proc3....
# Select one month
month_ds = cpcuf_ds.sel(
time=slice("2023-01-01", "2023-02-01")
)
print(month_ds)<xarray.Dataset>
Dimensions: (time: 2764587)
Coordinates:
* time (time) datetime64[ns] 2023-01-01T00:00:00.6799999...
Data variables:
concentration (time) float32 dask.array<chunksize=(86400,), meta=np.ndarray>
qc_concentration (time) int32 dask.array<chunksize=(86400,), meta=np.ndarray>
concentration_qc_fixed (time) float32 dask.array<chunksize=(86400,), meta=np.ndarray>
Attributes: (12/21)
command_line: aoscpcufcorr -s sgp -f E13
Conventions: ARM-1.2
process_version: ingest-aoscpcufcorr-1.5-0.el7
dod_version: aoscpcuf-b1-4.0
site_id: sgp
platform_id: aoscpcuf
... ...
mentor_affiliation: BNL
input_datastreams: sgpaoscpcufE13.a1 : 1.2 : 20221230.000000
configuration_file: sgpaoscpcufE13.20190328.160000.config
AOS_DAC_type: D
doi: 10.5439/1378933
history: created by user dsmgr on machine prod-proc3....
fig, axes = plt.subplots(
nrows=3,
ncols=1,
figsize=(16, 10),
sharex=True
)
month_ds["concentration"].plot(ax=axes[0])
axes[0].set_title("Original Concentration")
axes[0].set_ylabel("Concentration")
month_ds["qc_concentration"].plot(ax=axes[1])
axes[1].set_title("QC Concentration")
axes[1].set_ylabel("QC")
month_ds["concentration_qc_fixed"].plot(ax=axes[2])
axes[2].set_title("QC-fixed Concentration")
axes[2].set_ylabel("QC-fixed Concentration")
axes[2].set_xlabel("Time")
for ax in axes:
ax.grid(True)
plt.tight_layout()
plt.show()

day_ds = cpcuf_ds.sel(
time=slice("2023-01-05", "2023-01-07")
)
print(day_ds)<xarray.Dataset>
Dimensions: (time: 259186)
Coordinates:
* time (time) datetime64[ns] 2023-01-05T00:00:00.190000 ...
Data variables:
concentration (time) float32 dask.array<chunksize=(86399,), meta=np.ndarray>
qc_concentration (time) int32 dask.array<chunksize=(86399,), meta=np.ndarray>
concentration_qc_fixed (time) float32 dask.array<chunksize=(86399,), meta=np.ndarray>
Attributes: (12/21)
command_line: aoscpcufcorr -s sgp -f E13
Conventions: ARM-1.2
process_version: ingest-aoscpcufcorr-1.5-0.el7
dod_version: aoscpcuf-b1-4.0
site_id: sgp
platform_id: aoscpcuf
... ...
mentor_affiliation: BNL
input_datastreams: sgpaoscpcufE13.a1 : 1.2 : 20221230.000000
configuration_file: sgpaoscpcufE13.20190328.160000.config
AOS_DAC_type: D
doi: 10.5439/1378933
history: created by user dsmgr on machine prod-proc3....
fig, axes = plt.subplots(
nrows=3,
ncols=1,
figsize=(16, 10),
sharex=True
)
day_ds["concentration"].plot(ax=axes[0])
axes[0].set_title("Original Concentration")
axes[0].set_ylabel("Concentration")
day_ds["qc_concentration"].plot(ax=axes[1])
axes[1].set_title("QC Concentration")
axes[1].set_ylabel("QC")
day_ds["concentration_qc_fixed"].plot(ax=axes[2])
axes[2].set_title("QC-fixed Concentration")
axes[2].set_ylabel("QC-fixed Concentration")
axes[2].set_xlabel("Time")
for ax in axes:
ax.grid(True)
plt.tight_layout()
plt.show()
cpcuf_ds["time_Local"] = cpcuf_ds["time"] - pd.Timedelta(hours=6)
cpcuf_ds["time_Local"].attrs["note"] = "UTC time minus 6 hours"print(cpcuf_ds["time"].isel(time=slice(0, 5)).values)
print(cpcuf_ds["time_Local"].isel(time=slice(0, 5)).values)['2022-12-30T00:00:00.950000000' '2022-12-30T00:00:01.950000000'
'2022-12-30T00:00:02.940000000' '2022-12-30T00:00:03.950000000'
'2022-12-30T00:00:04.950000000']
['2022-12-29T18:00:00.950000000' '2022-12-29T18:00:01.950000000'
'2022-12-29T18:00:02.940000000' '2022-12-29T18:00:03.950000000'
'2022-12-29T18:00:04.950000000']
print(cpcuf_ds["time_Local"].attrs)
{'note': 'UTC time minus 6 hours'}
path = os.path.expanduser(
'~/DOE_ARM_SummerSchool/sgpaoscpcfE13.b1/sgpaoscpcfE13.b1*.nc'
)
cpcFINE_data_files = sorted(glob.glob(path))
cpcFINE_ds = xr.open_mfdataset(
cpcFINE_data_files,
combine="by_coords"
)
# Keep concentration only where qc_concentration == 0
# Usually qc == 0 means good data
# Keep original concentration
# Add QC-fixed concentration as a new xarray variable
cpcFINE_ds["concentration_qc_fixed"] = cpcFINE_ds["concentration"].where(
cpcFINE_ds["qc_concentration"] == 0
)
# Optional: keep only the variables you care about
cpcFINE_ds = cpcFINE_ds[
["concentration", "qc_concentration", "concentration_qc_fixed"]
]
print(cpcFINE_ds)<xarray.Dataset>
Dimensions: (time: 31764571)
Coordinates:
* time (time) datetime64[ns] 2022-12-30T00:00:00.950000 ...
Data variables:
concentration (time) float32 dask.array<chunksize=(86399,), meta=np.ndarray>
qc_concentration (time) int32 dask.array<chunksize=(86399,), meta=np.ndarray>
concentration_qc_fixed (time) float32 dask.array<chunksize=(86399,), meta=np.ndarray>
Attributes: (12/21)
command_line: aoscpcfcorr -s sgp -f E13 -b 20221205 -e 202...
Conventions: ARM-1.2
process_version: ingest-aoscpcfcorr-2.7-0.el7
dod_version: aoscpcf-b1-4.0
site_id: sgp
platform_id: aoscpcf
... ...
mentor_affiliation: BNL
input_datastreams: sgpaoscpcfE13.a1 : 1.1 : 20221230.000000
configuration_file: sgpaoscpcfE13.20190328.160000.config
AOS_DAC_type: D
doi: 10.5439/1352536
history: created by user yahmad on machine prod-proc5...
cpcFINE_ds["time_Local"] = cpcFINE_ds["time"] - pd.Timedelta(hours=6)
cpcFINE_ds["time_Local"].attrs["note"] = "UTC time minus 6 hours"print(cpcFINE_ds["time"].isel(time=slice(0, 5)).values)
print(cpcFINE_ds["time_Local"].isel(time=slice(0, 5)).values)['2022-12-30T00:00:00.950000000' '2022-12-30T00:00:01.960000000'
'2022-12-30T00:00:02.950000000' '2022-12-30T00:00:03.950000000'
'2022-12-30T00:00:04.950000000']
['2022-12-29T18:00:00.950000000' '2022-12-29T18:00:01.960000000'
'2022-12-29T18:00:02.950000000' '2022-12-29T18:00:03.950000000'
'2022-12-29T18:00:04.950000000']
cpcFINE_local = cpcFINE_ds.swap_dims({"time": "time_Local"})
cpcuf_local = cpcuf_ds.swap_dims({"time": "time_Local"})cpcFINE_local = cpcFINE_local.rename({
"concentration": "concentration_FINE",
"concentration_qc_fixed": "concentration_FINE_qc_fixed",
"qc_concentration": "qc_concentration_FINE"
})
cpcFINE_local_clean = (
cpcFINE_local[["concentration_FINE_qc_fixed"]]
.drop_vars(["time"], errors="ignore")
)
cpcuf_local = cpcuf_local.rename({
"concentration": "concentration_UF",
"qc_concentration": "qc_concentration_UF",
"concentration_qc_fixed": "concentration_UF_qc_fixed"
})
cpcuf_local_clean = (
cpcuf_local[["concentration_UF_qc_fixed"]]
.drop_vars(["time"], errors="ignore")
)
# 5-minute averages for each dataset
cpcFINE_merge = cpcFINE_local_clean.resample(time_Local="5min").mean()
cpcuf_merge = cpcuf_local_clean.resample(time_Local="5min").mean()
print("all done")all done
fine_times = cpcFINE_merge["time_Local"].isel(time_Local=slice(0, 5)).values
uf_times = cpcuf_merge["time_Local"].isel(time_Local=slice(0, 5)).values
for i, (fine_t, uf_t) in enumerate(zip(fine_times, uf_times)):
print(i, "FINE:", fine_t, "| UF:", uf_t)0 FINE: 2022-12-29T18:00:00.000000000 | UF: 2022-12-29T18:00:00.000000000
1 FINE: 2022-12-29T18:05:00.000000000 | UF: 2022-12-29T18:05:00.000000000
2 FINE: 2022-12-29T18:10:00.000000000 | UF: 2022-12-29T18:10:00.000000000
3 FINE: 2022-12-29T18:15:00.000000000 | UF: 2022-12-29T18:15:00.000000000
4 FINE: 2022-12-29T18:20:00.000000000 | UF: 2022-12-29T18:20:00.000000000
#day_mergeLoading...
#plt.figure(figsize=(14, 5))
#day_merge["concentration_UF_qc_fixed"].plot()
#plt.xlabel("Local Time")
#plt.ylabel("UF QC-fixed Concentration")
#plt.title("UF QC-fixed Concentration, 5-minute average")
#plt.grid(True)
#plt.tight_layout()
#plt.show()
merged_cpc_5min = xr.merge(
[cpcFINE_merge, cpcuf_merge],
join="inner"
)
print(merged_cpc_5min)<xarray.Dataset>
Dimensions: (time_Local: 105984)
Coordinates:
* time_Local (time_Local) datetime64[ns] 2022-12-29T18:00...
Data variables:
concentration_FINE_qc_fixed (time_Local) float32 dask.array<chunksize=(1,), meta=np.ndarray>
concentration_UF_qc_fixed (time_Local) float32 dask.array<chunksize=(1,), meta=np.ndarray>
merged_cpc_5min["UF_minus_FINE"] = (
merged_cpc_5min["concentration_UF_qc_fixed"]
- merged_cpc_5min["concentration_FINE_qc_fixed"]
)
merged_cpc_2023 = merged_cpc_5min.sel(
time_Local=slice("2023-01-01", "2024-01-01")
)
print(merged_cpc_2023)<xarray.Dataset>
Dimensions: (time_Local: 105336)
Coordinates:
* time_Local (time_Local) datetime64[ns] 2023-01-01 ... 2...
Data variables:
concentration_FINE_qc_fixed (time_Local) float32 dask.array<chunksize=(1,), meta=np.ndarray>
concentration_UF_qc_fixed (time_Local) float32 dask.array<chunksize=(1,), meta=np.ndarray>
UF_minus_FINE (time_Local) float32 dask.array<chunksize=(1,), meta=np.ndarray>
merged_cpc_2023_df = (
merged_cpc_2023[["UF_minus_FINE"]]
.chunk({"time_Local": -1})
.compute()
.to_dataframe()
.reset_index()
)merged_cpc_2023_df.to_csv("merged_cpc_2023_UF_minus_FINE5min.csv", index=False)