Skip to content

Commit 9ad1921

Browse files
authored
Merge branch 'develop' into ic-generation
2 parents 279944a + badf46f commit 9ad1921

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+1310
-296
lines changed

.github/workflows/pkgdown.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ jobs:
1616
runs-on: ubuntu-latest
1717
env:
1818
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
19+
PECAN_GIT_BRANCH: ${{ github.event_name == 'pull_request' && github.base_ref || github.ref_name }}
1920

2021
container:
2122
image: pecan/depends:develop
@@ -24,7 +25,7 @@ jobs:
2425
# Checkout source code
2526
- uses: actions/checkout@v4
2627

27-
# Install pkgdown
28+
# Install dependencies
2829
- name: Install dependencies
2930
run: Rscript -e 'install.packages("pkgdown")'
3031

CHANGELOG.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ section for the next release.
1111

1212
* Add function `clip_and_save_raster_file()` for subsetting rasters to match a polygon of interest (#3537).
1313
* Add CH4 and N2O to standard_vars in PEcAn.utils
14-
14+
* New function `sat_vapor_pressure()` added for computing saturation vapor pressure from temperature using various methods.
15+
*
1516
## [1.9.0] - 2025-05-25
1617

1718
### Added
@@ -33,6 +34,7 @@ section for the next release.
3334
- **`soilgrids_ic_process`**: A function to extract, process, and generate ensemble members from SoilGrids250m data.
3435
- **`preprocess_soilgrids_data`**: A helper function to handle missing values and ensure data integrity during preprocessing.
3536
- **`generate_soilgrids_ensemble`**: A function to create ensemble members for a site based on processed soil carbon data.
37+
- `extract.nc.ERA5()` and `met2CF.ERA5` now supports both ensemble and reanalysis data processing .
3638

3739
### Fixed
3840
- api to correctly use x_var from request in plotResults #3528
@@ -58,6 +60,7 @@ section for the next release.
5860
* Modules `PEcAn.allometry`, `PEcAn.assim.batch`, `PEcAn.data.mining`, `PEcAn.emulator`, `PEcAn.MA`, `PEcAn.photosynthesis`, `PEcAn.priors`, and `PEcAn.RTM`.
5961
- Renamed master branch to main
6062
- `PEcAn.all::pecan_version()` now reports commit hashes as well as version numbers for each installed package.
63+
- `download.ERA5_cds` now uses the R package ecmwfr (replacing python dependency of cdsapi via reticulate), enabling direct NetCDF downloads; and made flexible for both reanalysis and ensemble data product.
6164

6265
### Removed
6366

apps/api/R/entrypoint.R

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,10 @@ root$mount("/api/runs", runs_pr)
6969
runs_pr <- plumber::Plumber$new("available-models.R")
7070
root$mount("/api/availableModels", runs_pr)
7171

72+
# The endpoints mounted here are related to details of PEcAn posteriors
73+
runs_pr <- plumber::Plumber$new("posteriors.R")
74+
root$mount("/api/posteriors", runs_pr)
75+
7276
# set swagger documentation
7377
root$setApiSpec("../pecanapi-spec.yml")
7478

apps/api/R/posteriors.R

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
library(dplyr)
2+
3+
#' Search for Posteriors containing wildcards for filtering
4+
#' @param pft_id PFT Id (character)
5+
#' @param offset
6+
#' @param limit
7+
#' @return Information about Posteriors based on pft
8+
#' @author Nihar Sanda
9+
#* @get /
10+
searchPosteriors <- function(req, pft_id = NA, host_id = NA,
11+
offset = 0, limit = 50, res) {
12+
if (!limit %in% c(10, 20, 50, 100, 500)) {
13+
res$status <- 400
14+
return(list(error = "limit parameter must be 10, 20, 50, 100, or 500"))
15+
}
16+
17+
posteriors <- tbl(global_db_pool, "posteriors") %>%
18+
select(everything())
19+
20+
posteriors <- tbl(global_db_pool, "dbfiles") %>%
21+
select(file_name,
22+
file_path,
23+
container_type,
24+
id = container_id,
25+
machine_id) %>%
26+
inner_join(posteriors, by = "id") %>%
27+
filter(container_type == "Posterior") %>%
28+
select(-container_type)
29+
30+
posteriors <- tbl(global_db_pool, "machines") %>%
31+
select(hostname, machine_id = id) %>%
32+
inner_join(posteriors, by = "machine_id")
33+
34+
posteriors <- tbl(global_db_pool, "pfts") %>%
35+
select(pft_name = name, pft_id = id) %>%
36+
inner_join(posteriors, by = "pft_id")
37+
38+
if (!is.na(pft_id)) {
39+
posteriors <- posteriors %>%
40+
filter(pft_id == !!pft_id)
41+
}
42+
43+
if (!is.na(host_id)) {
44+
posteriors <- posteriors %>%
45+
filter(machine_id == !!host_id)
46+
}
47+
48+
qry_res <- posteriors %>%
49+
select(-pft_id, -machine_id) %>%
50+
distinct() %>%
51+
arrange(id) %>%
52+
collect()
53+
54+
if (nrow(qry_res) == 0 || as.numeric(offset) >= nrow(qry_res)) {
55+
res$status <- 404
56+
return(list(error = "Posterior(s) not found"))
57+
} else {
58+
has_next <- FALSE
59+
has_prev <- FALSE
60+
if (nrow(qry_res) > (as.numeric(offset) + as.numeric(limit))) {
61+
has_next <- TRUE
62+
}
63+
if (as.numeric(offset) != 0) {
64+
has_prev <- TRUE
65+
}
66+
67+
start_idx <- as.numeric(offset) + 1
68+
end_idx <- min((as.numeric(offset) + as.numeric(limit)), nrow(qry_res))
69+
qry_res <- qry_res[start_idx:end_idx, ]
70+
71+
result <- list(posteriors = qry_res)
72+
result$count <- nrow(qry_res)
73+
if (has_next) {
74+
if (grepl("offset=", req$QUERY_STRING, fixed = TRUE)) {
75+
result$next_page <- paste0(
76+
req$rook.url_scheme, "://",
77+
req$HTTP_HOST,
78+
"/api/posteriors",
79+
req$PATH_INFO,
80+
substr(req$QUERY_STRING,
81+
0,
82+
stringr::str_locate(req$QUERY_STRING, "offset=")[[2]]),
83+
(as.numeric(limit) + as.numeric(offset)),
84+
"&limit=",
85+
limit
86+
)
87+
} else {
88+
result$next_page <- paste0(
89+
req$rook.url_scheme, "://",
90+
req$HTTP_HOST,
91+
"/api/posteriors",
92+
req$PATH_INFO,
93+
substr(req$QUERY_STRING,
94+
0,
95+
stringr::str_locate(req$QUERY_STRING, "limit=")[[2]] - 6),
96+
"offset=",
97+
(as.numeric(limit) + as.numeric(offset)),
98+
"&limit=",
99+
limit
100+
)
101+
}
102+
}
103+
if (has_prev) {
104+
result$prev_page <- paste0(
105+
req$rook.url_scheme, "://",
106+
req$HTTP_HOST,
107+
"/api/workflows",
108+
req$PATH_INFO,
109+
substr(req$QUERY_STRING,
110+
0,
111+
stringr::str_locate(req$QUERY_STRING, "offset=")[[2]]),
112+
max(0, (as.numeric(offset) - as.numeric(limit))),
113+
"&limit=",
114+
limit
115+
)
116+
}
117+
118+
return(result)
119+
}
120+
}
121+
122+
################################################################################
123+
124+
#' Download the posterior specified by the id
125+
#' @param id Posterior id (character)
126+
#' @param filename Optional filename specified if the id points to a folder
127+
#' instead of file (character). If this is passed with an id that actually
128+
#' points to a file, this name will be ignored
129+
#' @return Posterior file specified by user
130+
#' @author Nihar Sanda
131+
#* @serializer contentType list(type="application/octet-stream")
132+
#* @get /<posterior_id>
133+
downloadPosterior <- function(posterior_id, filename = "", req, res) {
134+
db_hostid <- PEcAn.DB::dbHostInfo(global_db_pool)$hostid
135+
136+
# This is just for temporary testing due to the existing issue in dbHostInfo()
137+
db_hostid <- ifelse(db_hostid == 99, 99000000001, db_hostid)
138+
139+
posterior <- tbl(global_db_pool, "dbfiles") %>%
140+
select(file_name, file_path, container_id, machine_id, container_type) %>%
141+
filter(machine_id == !!db_hostid) %>%
142+
filter(container_type == "Posterior") %>%
143+
filter(container_id == !!posterior_id) %>%
144+
collect()
145+
146+
if (filename != "") {
147+
posterior <- posterior %>%
148+
filter(file_name == !!filename)
149+
}
150+
151+
if (nrow(posterior) == 0) {
152+
res$status <- 404
153+
return("Posterior not found")
154+
}
155+
156+
# Generate the full file path using the file_path & file_name
157+
filepath <- file.path(posterior$file_path, posterior$file_name)
158+
159+
if (length(filepath) > 1 || dir.exists(filepath)) {
160+
# Don't know which file to send. Return 400 Bad Request error
161+
# TODO provide an endpoint to list the available files from one posterior
162+
# (maybe `/posteriors/{posterior_id}/files`?)
163+
res$status <- 400
164+
return("Multiple matches. Please specify filename")
165+
}
166+
167+
# If the file doesn't exist, return 404 error
168+
if (!file.exists(filepath)) {
169+
res$status <- 404
170+
return("Posterior file not found")
171+
}
172+
173+
# Read the data in binary form & return it
174+
bin <- readBin(filepath, "raw", n = file.info(filepath)$size)
175+
return(bin)
176+
}

apps/api/pecanapi-spec.yml

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ tags:
4141
description: Everything about PEcAn PFTs (Plant Functional Types)
4242
- name: inputs
4343
description: Everything about PEcAn inputs
44+
- name: posteriors
45+
description: Everything about PEcAn posteriors
4446

4547
#####################################################################################################################
4648
##################################################### API Endpoints #################################################
@@ -992,6 +994,123 @@ paths:
992994
description: Access forbidden
993995
'404':
994996
description: Run data not found
997+
998+
/api/posteriors/:
999+
get:
1000+
tags:
1001+
- posteriors
1002+
summary: Search for the posteriors
1003+
parameters:
1004+
- in: query
1005+
name: pft_id
1006+
description: If provided, returns all posteriors for the provided pft_id
1007+
required: false
1008+
schema:
1009+
type: string
1010+
- in: query
1011+
name: host_id
1012+
description: If provided, returns all posteriors for the provided host_id
1013+
required: false
1014+
schema:
1015+
type: string
1016+
- in: query
1017+
name: offset
1018+
description: The number of posteriors to skip before starting to collect the result set.
1019+
schema:
1020+
type: integer
1021+
minimum: 0
1022+
default: 0
1023+
required: false
1024+
- in: query
1025+
name: limit
1026+
description: The number of posteriors to return.
1027+
schema:
1028+
type: integer
1029+
default: 50
1030+
enum:
1031+
- 10
1032+
- 20
1033+
- 50
1034+
- 100
1035+
- 500
1036+
required: false
1037+
responses:
1038+
'200':
1039+
description: List of posteriors
1040+
content:
1041+
application/json:
1042+
schema:
1043+
type: object
1044+
properties:
1045+
inputs:
1046+
type: array
1047+
items:
1048+
type: object
1049+
properties:
1050+
id:
1051+
type: string
1052+
file_name:
1053+
type: string
1054+
file_path:
1055+
type: string
1056+
pft_name:
1057+
type: string
1058+
tag:
1059+
type: string
1060+
hostname:
1061+
type: string
1062+
start_date:
1063+
type: string
1064+
end_date:
1065+
type: string
1066+
count:
1067+
type: integer
1068+
next_page:
1069+
type: string
1070+
prev_page:
1071+
type: string
1072+
1073+
'401':
1074+
description: Authentication required
1075+
'403':
1076+
description: Access forbidden
1077+
'404':
1078+
description: Workflows not found
1079+
1080+
/api/posteriors/{posterior_id}:
1081+
get:
1082+
tags:
1083+
- posteriors
1084+
summary: Download a desired PEcAn posterior file
1085+
parameters:
1086+
- in: path
1087+
name: posterior_id
1088+
description: ID of the PEcAn Posterior to be downloaded
1089+
required: true
1090+
schema:
1091+
type: string
1092+
- in: query
1093+
name: filename
1094+
description: Optional filename specified if the id points to a folder instead of file
1095+
required: false
1096+
schema:
1097+
type: string
1098+
responses:
1099+
'200':
1100+
description: Contents of the desired posterior file
1101+
content:
1102+
application/octet-stream:
1103+
schema:
1104+
type: string
1105+
format: binary
1106+
'400':
1107+
description: Bad request. Posterior ID points to directory & filename is not specified
1108+
'401':
1109+
description: Authentication required
1110+
'403':
1111+
description: Access forbidden
1112+
1113+
9951114
#####################################################################################################################
9961115
###################################################### Components ###################################################
9971116
#####################################################################################################################

base/all/DESCRIPTION

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ Description: The Predictive Ecosystem Carbon Analyzer
4343
PEcAn is to streamline the interaction between data and
4444
models, and to improve the efficacy of scientific
4545
investigation.
46+
URL: https://pecanproject.github.io
47+
BugReports: https://github.com/PecanProject/pecan/issues
4648
Depends:
4749
PEcAn.DB,
4850
PEcAn.settings,

base/db/DESCRIPTION

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ Description: The Predictive Ecosystem Carbon Analyzer (PEcAn) is a scientific
3939
model parameterization, execution, and analysis. The goal of PECAn is to
4040
streamline the interaction between data and models, and to improve the
4141
efficacy of scientific investigation.
42+
URL: https://pecanproject.github.io
43+
BugReports: https://github.com/PecanProject/pecan/issues
4244
Imports:
4345
DBI,
4446
dbplyr (>= 2.4.0),

base/db/man/PEcAn.DB-package.Rd

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)