Skip to content

change output folder based on run type #57

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
harshavemula-ua merged 3 commits into from
Feb 19, 2026
Merged

change output folder based on run type #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
39cd2ba
change output folder based on run type
quinnylee Feb 11, 2026
3353c56
added -p flag to mkdir
quinnylee Feb 11, 2026
2455ed0
update pytest
quinnylee Feb 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions src/forcingprocessor/processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -901,11 +901,14 @@ def prep_ngen_data(conf):
if output_path == "":
output_path = os.path.join(os.getcwd(),datentime)
output_path = Path(output_path)
forcing_path = Path(output_path, 'forcings')
if data_source == "channel_routing":
forcing_path = Path(output_path, 'outputs', 'ngen')
else:
forcing_path = Path(output_path, 'forcings')
meta_path = Path(output_path, 'metadata')
metaf_path = Path(output_path, 'metadata','forcings_metadata')
if not os.path.exists(output_path): os.system(f"mkdir {output_path}")
if not os.path.exists(forcing_path): os.system(f"mkdir {forcing_path}")
if not os.path.exists(forcing_path): os.system(f"mkdir -p {forcing_path}")
if not os.path.exists(meta_path): os.system(f"mkdir {meta_path}")
if not os.path.exists(metaf_path): os.system(f"mkdir {metaf_path}")
conf_path = Path
Expand Down
38 changes: 19 additions & 19 deletions tests/test_channelrouting.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
yesterday = yesterday.strftime('%Y%m%d')
test_dir = Path(__file__).parent
data_dir = (test_dir/'data').resolve()
forcings_dir = (data_dir/'forcings').resolve()
forcings_dir = (data_dir/'outputs/ngen').resolve()
pwd = Path.cwd()
if os.path.exists(data_dir):
os.system(f"rm -rf {data_dir}")
Expand Down Expand Up @@ -81,7 +81,7 @@ def test_nomads_prod(download_weight_file,clean_forcings_metadata_dirs):
conf['run']['collect_stats'] = True # test metadata generation once
prep_ngen_data(conf)
conf['run']['collect_stats'] = False
assert_file=(data_dir/"forcings/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert_file=(data_dir/"outputs/ngen/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -91,7 +91,7 @@ def test_nwm_google_apis(download_weight_file,clean_forcings_metadata_dirs):
nwmurl_conf["urlbaseinput"] = 3
generate_nwmfiles(nwmurl_conf)
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert_file=(data_dir/"outputs/ngen/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -101,7 +101,7 @@ def test_google_cloud_storage(download_weight_file,clean_forcings_metadata_dirs)
nwmurl_conf["urlbaseinput"] = 4
generate_nwmfiles(nwmurl_conf)
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert_file=(data_dir/"outputs/ngen/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -110,7 +110,7 @@ def test_gs(download_weight_file,clean_forcings_metadata_dirs):
nwmurl_conf['end_date'] = TODAY_START
nwmurl_conf["urlbaseinput"] = 5
generate_nwmfiles(nwmurl_conf)
assert_file=(data_dir/"forcings/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert_file=(data_dir/"outputs/ngen/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
prep_ngen_data(conf)
assert assert_file.exists()
os.remove(assert_file)
Expand All @@ -121,7 +121,7 @@ def test_gcs(download_weight_file,clean_forcings_metadata_dirs):
nwmurl_conf["urlbaseinput"] = 6
generate_nwmfiles(nwmurl_conf)
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert_file=(data_dir/"outputs/ngen/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -131,7 +131,7 @@ def test_noaa_nwm_pds_https(download_weight_file,clean_forcings_metadata_dirs):
nwmurl_conf["urlbaseinput"] = 7
generate_nwmfiles(nwmurl_conf)
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert_file=(data_dir/"outputs/ngen/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -142,7 +142,7 @@ def test_noaa_nwm_pds_https_short_range(download_weight_file,clean_forcings_meta
nwmurl_conf["runinput"] = 1
generate_nwmfiles(nwmurl_conf)
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert_file=(data_dir/"outputs/ngen/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -154,7 +154,7 @@ def test_noaa_nwm_pds_https_medium_range(download_weight_file,clean_forcings_met
generate_nwmfiles(nwmurl_conf)
prep_ngen_data(conf)
assert_file=(
data_dir/"forcings/ngen.t00z.medium_range_mem1.channel_routing.f001_f001.nc").resolve()
data_dir/"outputs/ngen/ngen.t00z.medium_range_mem1.channel_routing.f001_f001.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -166,7 +166,7 @@ def test_noaa_nwm_pds_https_analysis_assim(download_weight_file,clean_forcings_m
generate_nwmfiles(nwmurl_conf)
prep_ngen_data(conf)
assert_file=(
data_dir/"forcings/ngen.t00z.analysis_assim.channel_routing.tm01_tm01.nc").resolve()
data_dir/"outputs/ngen/ngen.t00z.analysis_assim.channel_routing.tm01_tm01.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -190,7 +190,7 @@ def test_noaa_nwm_pds_https_analysis_assim_extend(
else:
raise
assert_file=(
data_dir/"forcings/ngen.t16z.analysis_assim_extend.channel_routing.tm01_tm01.nc").resolve()
data_dir/"outputs/ngen/ngen.t16z.analysis_assim_extend.channel_routing.tm01_tm01.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -202,7 +202,7 @@ def test_noaa_nwm_pds_s3(download_weight_file,clean_forcings_metadata_dirs):
nwmurl_conf["fcst_cycle"] = [0]
generate_nwmfiles(nwmurl_conf)
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert_file=(data_dir/"outputs/ngen/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -211,7 +211,7 @@ def test_retro_2_1_https(download_weight_file,clean_forcings_metadata_dirs):
nwmurl_conf_retro["urlbaseinput"] = 1
generate_nwmfiles(nwmurl_conf_retro)
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/qlaterals.nc").resolve()
assert_file=(data_dir/"outputs/ngen/qlaterals.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -221,7 +221,7 @@ def test_retro_2_1_s3(download_weight_file,clean_forcings_metadata_dirs):
nwmurl_conf_retro["urlbaseinput"] = 2
generate_nwmfiles(nwmurl_conf_retro)
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/qlaterals.nc").resolve()
assert_file=(data_dir/"outputs/ngen/qlaterals.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -230,7 +230,7 @@ def test_retro_3_0(download_weight_file,clean_forcings_metadata_dirs):
nwmurl_conf_retro["urlbaseinput"] = 4
generate_nwmfiles(nwmurl_conf_retro)
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/qlaterals.nc").resolve()
assert_file=(data_dir/"outputs/ngen/qlaterals.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Expand All @@ -255,31 +255,31 @@ def test_csv_output_type(download_weight_file,clean_forcings_metadata_dirs):
generate_nwmfiles(nwmurl_conf)
conf['storage']['output_file_type'] = ["csv"]
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/nex-1555523.csv").resolve()
assert_file=(data_dir/"outputs/ngen/nex-1555523.csv").resolve()
assert assert_file.exists()
os.remove(assert_file)

def test_parquet_output_type(download_weight_file,clean_forcings_metadata_dirs):
generate_nwmfiles(nwmurl_conf)
conf['storage']['output_file_type'] = ["parquet"]
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/nex-1555523.parquet").resolve()
assert_file=(data_dir/"outputs/ngen/nex-1555523.parquet").resolve()
assert assert_file.exists()
os.remove(assert_file)

def test_tar_output_type(download_weight_file,clean_forcings_metadata_dirs):
generate_nwmfiles(nwmurl_conf)
conf['storage']['output_file_type'] = ["tar"]
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/1_forcings.tar.gz").resolve()
assert_file=(data_dir/"outputs/ngen/1_forcings.tar.gz").resolve()
assert assert_file.exists()
os.remove(assert_file)

def test_netcdf_output_type(download_weight_file,clean_forcings_metadata_dirs):
generate_nwmfiles(nwmurl_conf)
conf['storage']['output_file_type'] = ["netcdf"]
prep_ngen_data(conf)
assert_file=(data_dir/"forcings/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert_file=(data_dir/"outputs/ngen/ngen.t00z.short_range.channel_routing.f001_f001.nc").resolve()
assert assert_file.exists()
os.remove(assert_file)

Loading