Skip to content
Merged
Show file tree
Hide file tree
Changes from 122 commits
Commits
Show all changes
160 commits
Select commit Hold shift + click to select a range
155bb91
draft project/services api
pcrespov Feb 26, 2025
bcf3fa4
draft test and api in catalog
pcrespov Feb 26, 2025
1ff3970
updates OAS
pcrespov Feb 26, 2025
36d0150
draft
pcrespov Feb 26, 2025
c110249
draft 2
pcrespov Feb 26, 2025
1f9da9e
draft 3
pcrespov Feb 27, 2025
504a11d
fix tests
pcrespov Feb 27, 2025
19cc9d1
expanding test
pcrespov Feb 27, 2025
50f2de1
update
pcrespov Feb 27, 2025
d2f8ec9
connect to rpc
pcrespov Feb 27, 2025
f5a7c40
updates listing
pcrespov Feb 27, 2025
43ae5b0
fix tests
pcrespov Feb 27, 2025
9479efd
cleanup
pcrespov Feb 27, 2025
dfd365e
drop api changes
pcrespov Feb 27, 2025
13013e3
items
pcrespov Feb 27, 2025
47d96c6
other user fixture
pcrespov Feb 27, 2025
3451d57
fix tests
pcrespov Feb 28, 2025
e2bfac8
drafts test
pcrespov Feb 28, 2025
18c0dd5
fixes models
pcrespov Feb 28, 2025
b3aed7d
minor oas changes
pcrespov Feb 28, 2025
f8b77f6
getServices
odeimaiz Feb 28, 2025
52319ab
adds get_project_services handle
pcrespov Feb 28, 2025
99e999c
drafts test
pcrespov Feb 28, 2025
ead3316
implementing get_project_nodes_services
pcrespov Feb 28, 2025
a79eb6d
adds test
pcrespov Feb 28, 2025
4c5c868
fixes mypy
pcrespov Feb 28, 2025
e663629
fixes pylint
pcrespov Feb 28, 2025
a6f3051
fixes tests
pcrespov Feb 28, 2025
02bb0fa
updates OAS
pcrespov Feb 28, 2025
323ab38
services/webserver api version: 0.60.0 → 0.61.0
pcrespov Feb 28, 2025
d1ae3d9
adds access check
pcrespov Feb 28, 2025
b31ab1f
accepts no ownership
pcrespov Feb 28, 2025
f5c9a28
updates OAS
pcrespov Feb 28, 2025
31d0f11
cleanup
pcrespov Feb 28, 2025
ea7a012
extends test
pcrespov Feb 28, 2025
8183622
fixes pylint
pcrespov Feb 28, 2025
1555760
stage
odeimaiz Mar 3, 2025
fd2a4d6
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 3, 2025
924c083
Merge branch 'master' into is6201/catalog-history-alternative
pcrespov Mar 3, 2025
056bedc
fix pylint
pcrespov Mar 3, 2025
0510b0b
fix sonar
pcrespov Mar 3, 2025
c53cab9
@GitHK review: validate on server side
pcrespov Mar 3, 2025
5131518
Merge branch 'master' into is6201/catalog-history-alternative
pcrespov Mar 3, 2025
6797b92
Merge branch 'master' into is6201/catalog-history-alternative
pcrespov Mar 3, 2025
d09fc71
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 4, 2025
b4035d5
@sanderegg review: rename error
pcrespov Mar 4, 2025
6b02ba2
Merge branch 'master' into is6201/catalog-history-alternative
pcrespov Mar 4, 2025
b7e4327
Merge branch 'master' into is6201/catalog-history-alternative
pcrespov Mar 4, 2025
f4183c9
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 4, 2025
57c4848
Merge branch 'is6201/catalog-history-alternative' of github.com:pcres…
odeimaiz Mar 4, 2025
b328954
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 4, 2025
97cc41c
minor
odeimaiz Mar 4, 2025
9afd568
logic to services
odeimaiz Mar 4, 2025
57fcf39
unused
odeimaiz Mar 4, 2025
a95caed
minor
odeimaiz Mar 4, 2025
1c542da
remove history, it will be retired
odeimaiz Mar 4, 2025
a48e10b
blocked card
odeimaiz Mar 5, 2025
9a49b3a
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 5, 2025
553c9ed
study services to card
odeimaiz Mar 5, 2025
9d2ca61
minor
odeimaiz Mar 5, 2025
d7987e8
getOne -> fetch
odeimaiz Mar 5, 2025
36765b4
[skip ci] avoid cache
odeimaiz Mar 5, 2025
7e842ab
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 6, 2025
b61172d
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 6, 2025
64565ed
[skip ci] unused
odeimaiz Mar 6, 2025
4795245
Merge branch 'enh/new-batch-get-services' of github.com:odeimaiz/ospa…
odeimaiz Mar 6, 2025
7eb923e
minor
odeimaiz Mar 6, 2025
38f1d6f
[skip ci] minor
odeimaiz Mar 6, 2025
24036ac
[skip ci] more refactoring
odeimaiz Mar 6, 2025
155b9f6
refactor
odeimaiz Mar 6, 2025
d0ada28
delete history from latest
odeimaiz Mar 6, 2025
c106b0e
[skip ci] deprecated -> retired
odeimaiz Mar 6, 2025
734c5f5
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 6, 2025
cafef71
services as property
odeimaiz Mar 6, 2025
e154a4d
keep services
odeimaiz Mar 6, 2025
f444392
[skip ci] re-evaluateMenuButtons
odeimaiz Mar 6, 2025
8a2da31
[skip ci] rename
odeimaiz Mar 6, 2025
eb69530
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 7, 2025
d5e93fd
[skip ci] renaming
odeimaiz Mar 7, 2025
b7c241b
minor
odeimaiz Mar 7, 2025
9d3dfc5
[skip ci] Open renamings
odeimaiz Mar 7, 2025
35c31eb
getVersions
odeimaiz Mar 7, 2025
977e565
initPromises
odeimaiz Mar 7, 2025
dc86e83
getVersions in catalog
odeimaiz Mar 7, 2025
6ca5a87
refactoring
odeimaiz Mar 7, 2025
8efbc24
getLatest
odeimaiz Mar 7, 2025
1c9b00b
[skip ci] refactoring
odeimaiz Mar 7, 2025
04fae71
all moved to Services
odeimaiz Mar 7, 2025
c70e66f
[skip ci] minor
odeimaiz Mar 7, 2025
28256a7
[skip ci] more refactoring
odeimaiz Mar 7, 2025
bfa71e4
get latest
odeimaiz Mar 7, 2025
0dd8425
promised icon
odeimaiz Mar 7, 2025
ecd2694
update isUpdatable
odeimaiz Mar 7, 2025
626b08a
drops history from listing in catalog
pcrespov Mar 7, 2025
79ade3d
server
pcrespov Mar 7, 2025
5efaad2
reduce duplication
pcrespov Mar 7, 2025
d4d7d85
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 7, 2025
3232be1
[skip ci] history deprecated
odeimaiz Mar 7, 2025
cd506b1
minor
odeimaiz Mar 7, 2025
4d84a3a
[skip ci] refactor
odeimaiz Mar 7, 2025
cb9ec34
updatableNodeIds
odeimaiz Mar 7, 2025
d8b65ff
minor fix
odeimaiz Mar 7, 2025
ceff1fb
more refactoring
odeimaiz Mar 7, 2025
5806aee
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 10, 2025
75b46be
force fetch
odeimaiz Mar 10, 2025
68e8d04
minor
odeimaiz Mar 10, 2025
d5315f9
[skip ci] check cache's history
odeimaiz Mar 10, 2025
639fb3a
[skip ci] keep resourceData
odeimaiz Mar 10, 2025
d71d6a4
minor
odeimaiz Mar 10, 2025
f7cb8ad
minor
odeimaiz Mar 10, 2025
9c67e70
refactor versionsToSelectBox
odeimaiz Mar 10, 2025
6c9f264
minor
odeimaiz Mar 10, 2025
ca4c86d
[skip ci] populateVersionsSelectBox
odeimaiz Mar 10, 2025
dcd3933
unused
odeimaiz Mar 10, 2025
6da7f05
minors
odeimaiz Mar 10, 2025
afcd9ad
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 10, 2025
738168e
minors
odeimaiz Mar 10, 2025
200a9e1
text
odeimaiz Mar 10, 2025
1af9922
getStudyServicesMetadata
odeimaiz Mar 10, 2025
36d9543
minor
odeimaiz Mar 10, 2025
724334b
undo
odeimaiz Mar 10, 2025
93654e9
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 10, 2025
cdb60e4
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 10, 2025
cce3bc8
fixes pylint
pcrespov Mar 10, 2025
b4b386b
Merge branch 'master' into enh/new-batch-get-services
pcrespov Mar 10, 2025
35787c5
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 11, 2025
00f3cdc
pagesAdded event
odeimaiz Mar 11, 2025
f375c01
the "retired" info is in the history only
odeimaiz Mar 11, 2025
c1c08a8
comments
odeimaiz Mar 11, 2025
4a59c18
they will be included
odeimaiz Mar 11, 2025
669c51f
Merge branch 'master' into pr/odeimaiz/7292
pcrespov Mar 11, 2025
17e6726
Adds release field in LatestServiceGet items returned in GET services…
pcrespov Mar 11, 2025
5acbc1f
adapts tests
pcrespov Mar 11, 2025
26974d9
adapts tests
pcrespov Mar 11, 2025
b6e00e6
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 13, 2025
2bce07b
async guessIcon
odeimaiz Mar 13, 2025
1f991c5
release.retired
odeimaiz Mar 13, 2025
e8cbe37
[skip ci] renaming
odeimaiz Mar 13, 2025
2d3d7f7
[skip ci] renaming
odeimaiz Mar 13, 2025
b3a2673
minor fix
odeimaiz Mar 13, 2025
e4c077d
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 13, 2025
4287914
minor
odeimaiz Mar 13, 2025
070f6f7
minor
odeimaiz Mar 13, 2025
ccbac9e
minors
odeimaiz Mar 13, 2025
8878499
last changes
odeimaiz Mar 13, 2025
4b4c8e1
minor
odeimaiz Mar 13, 2025
5e10135
checkImageExists
odeimaiz Mar 13, 2025
5adc869
minor
odeimaiz Mar 13, 2025
b1e2563
fixes example
pcrespov Mar 13, 2025
e2ce790
Merge branch 'master' into pr/odeimaiz/7292
pcrespov Mar 13, 2025
f581607
fixes sonarcloud
pcrespov Mar 13, 2025
1151fc9
OAS update with example
pcrespov Mar 13, 2025
0185db7
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 13, 2025
9cd8644
Merge branch 'master' into enh/new-batch-get-services
odeimaiz Mar 14, 2025
16398f8
renaming
odeimaiz Mar 14, 2025
588c263
use history
odeimaiz Mar 14, 2025
923a763
refactor
odeimaiz Mar 14, 2025
dfce58a
minor fix
odeimaiz Mar 14, 2025
b3b02c3
reuse code
odeimaiz Mar 14, 2025
324b36a
refactor
odeimaiz Mar 14, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions api/specs/web-server/_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
from fastapi import APIRouter, Depends
from models_library.api_schemas_api_server.pricing_plans import ServicePricingPlanGet
from models_library.api_schemas_webserver.catalog import (
CatalogLatestServiceGet,
CatalogServiceGet,
CatalogServiceListItem,
CatalogServiceUpdate,
ServiceInputGet,
ServiceInputKey,
Expand Down Expand Up @@ -34,10 +34,9 @@

@router.get(
"/catalog/services/-/latest",
response_model=Page[CatalogServiceListItem],
response_model=Page[CatalogLatestServiceGet],
)
def list_services_latest(_query: Annotated[ListServiceParams, Depends()]):
pass
def list_services_latest(_query: Annotated[ListServiceParams, Depends()]): ...


@router.get(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,15 @@ class _BaseServiceGetV2(CatalogOutputSchema):

access_rights: dict[GroupID, ServiceGroupAccessRightsV2] | None

classifiers: list[str] | None = []
quality: dict[str, Any] = {}
classifiers: Annotated[
list[str] | None,
Field(default_factory=list),
] = DEFAULT_FACTORY

quality: Annotated[
dict[str, Any],
Field(default_factory=dict),
] = DEFAULT_FACTORY

model_config = ConfigDict(
extra="forbid",
Expand All @@ -212,6 +219,9 @@ class _BaseServiceGetV2(CatalogOutputSchema):
)


class LatestServiceGet(_BaseServiceGetV2): ...


class ServiceGetV2(_BaseServiceGetV2):
# Model used in catalog's rpc and rest interfaces
history: Annotated[
Expand Down Expand Up @@ -288,21 +298,9 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
)


class ServiceListItem(_BaseServiceGetV2):
history: Annotated[
list[ServiceRelease],
Field(
default_factory=list,
deprecated=True,
description="History will be replaced by current 'release' instead",
json_schema_extra={"default": []},
),
] = DEFAULT_FACTORY


PageRpcServicesGetV2: TypeAlias = PageRpc[
# WARNING: keep this definition in models_library and not in the RPC interface
ServiceListItem
LatestServiceGet
]

ServiceResourcesGet: TypeAlias = ServiceResourcesDict
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
ServiceResourcesGet: TypeAlias = api_schemas_catalog_services.ServiceResourcesGet


class CatalogServiceListItem(api_schemas_catalog_services.ServiceListItem):
class CatalogLatestServiceGet(api_schemas_catalog_services.LatestServiceGet):
inputs: ServiceInputsGetDict # type: ignore[assignment]
outputs: ServiceOutputsGetDict # type: ignore[assignment]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@

from models_library.api_schemas_catalog import CATALOG_RPC_NAMESPACE
from models_library.api_schemas_catalog.services import (
LatestServiceGet,
MyServiceGet,
ServiceGetV2,
ServiceListItem,
ServiceUpdateV2,
)
from models_library.products import ProductName
Expand Down Expand Up @@ -35,7 +35,7 @@ async def list_services_paginated( # pylint: disable=too-many-arguments
user_id: UserID,
limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE,
offset: NonNegativeInt = 0,
) -> PageRpc[ServiceListItem]:
) -> PageRpc[LatestServiceGet]:
"""
Raises:
ValidationError: on invalid arguments
Expand Down Expand Up @@ -63,9 +63,9 @@ async def _call(
product_name=product_name, user_id=user_id, limit=limit, offset=offset
)
assert ( # nosec
TypeAdapter(PageRpc[ServiceListItem]).validate_python(result) is not None
TypeAdapter(PageRpc[LatestServiceGet]).validate_python(result) is not None
)
return cast(PageRpc[ServiceListItem], result)
return cast(PageRpc[LatestServiceGet], result)


@log_decorator(_logger, level=logging.DEBUG)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ async def list_services_paginated(
) -> PageRpcServicesGetV2:
assert app.state.engine # nosec

total_count, items = await services_api.list_services_paginated(
total_count, items = await services_api.list_latest_services(
repo=ServicesRepository(app.state.engine),
director_api=get_director_api(app),
product_name=product_name,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def list_services_stmt(
combine_access_with_and: bool | None = True,
product_name: str | None = None,
) -> Select:
stmt = sa.select(SERVICES_META_DATA_COLS)
stmt = sa.select(*SERVICES_META_DATA_COLS)
if gids or execute_access or write_access:
conditions: list[Any] = []

Expand Down Expand Up @@ -135,7 +135,7 @@ def total_count_stmt(
)


def list_latest_services_with_history_stmt(
def list_latest_services_stmt(
*,
product_name: ProductName,
user_id: UserID,
Expand Down Expand Up @@ -174,7 +174,7 @@ def list_latest_services_with_history_stmt(
)

# get all information of latest's services listed in CTE
latest_query = (
latest_stmt = (
sa.select(
services_meta_data.c.key,
services_meta_data.c.version,
Expand Down Expand Up @@ -206,126 +206,26 @@ def list_latest_services_with_history_stmt(
.subquery("latest_sq")
)

# get history for every unique service-key in CTE
_accessible_sq = (
sa.select(
services_meta_data.c.key,
services_meta_data.c.version,
)
.distinct()
.select_from(
services_meta_data.join(
cte,
services_meta_data.c.key == cte.c.key,
)
# joins because access-rights might change per version
.join(
services_access_rights,
(services_meta_data.c.key == services_access_rights.c.key)
& (services_meta_data.c.version == services_access_rights.c.version)
& (services_access_rights.c.product_name == product_name),
)
.join(
user_to_groups,
(user_to_groups.c.gid == services_access_rights.c.gid)
& (user_to_groups.c.uid == user_id),
)
.outerjoin(
services_compatibility,
(services_meta_data.c.key == services_compatibility.c.key)
& (services_meta_data.c.version == services_compatibility.c.version),
)
)
.where(access_rights)
.subquery("accessible_sq")
)

history_subquery = (
sa.select(
services_meta_data.c.key,
services_meta_data.c.version,
services_meta_data.c.version_display,
services_meta_data.c.deprecated,
services_meta_data.c.created,
services_compatibility.c.custom_policy, # CompatiblePolicyDict | None
)
.select_from(
services_meta_data.join(
_accessible_sq,
(services_meta_data.c.key == _accessible_sq.c.key)
& (services_meta_data.c.version == _accessible_sq.c.version),
).outerjoin(
services_compatibility,
(services_meta_data.c.key == services_compatibility.c.key)
& (services_meta_data.c.version == services_compatibility.c.version),
)
)
.order_by(
services_meta_data.c.key,
sa.desc(_version(services_meta_data.c.version)), # latest version first
)
.subquery("history_sq")
)

return (
sa.select(
latest_query.c.key,
latest_query.c.version,
# display
latest_query.c.name,
latest_query.c.description,
latest_query.c.description_ui,
latest_query.c.thumbnail,
latest_query.c.icon,
latest_query.c.version_display,
# ownership
latest_query.c.owner_email,
# tags
latest_query.c.classifiers,
latest_query.c.quality,
# lifetime
latest_query.c.created,
latest_query.c.modified,
latest_query.c.deprecated,
# releases (NOTE: at some points we should limit this list?)
array_agg(
func.json_build_object(
"version",
history_subquery.c.version,
"version_display",
history_subquery.c.version_display,
"deprecated",
history_subquery.c.deprecated,
"created",
history_subquery.c.created,
"compatibility_policy", # NOTE: this is the `policy`
history_subquery.c.custom_policy,
)
).label("history"),
)
.join(
history_subquery,
latest_query.c.key == history_subquery.c.key,
)
.group_by(
history_subquery.c.key,
latest_query.c.key,
latest_query.c.version,
latest_query.c.owner_email,
latest_query.c.name,
latest_query.c.description,
latest_query.c.description_ui,
latest_query.c.thumbnail,
latest_query.c.icon,
latest_query.c.version_display,
latest_query.c.classifiers,
latest_query.c.created,
latest_query.c.modified,
latest_query.c.deprecated,
latest_query.c.quality,
)
.order_by(history_subquery.c.key)
)
return sa.select(
latest_stmt.c.key,
latest_stmt.c.version,
# display
latest_stmt.c.name,
latest_stmt.c.description,
latest_stmt.c.description_ui,
latest_stmt.c.thumbnail,
latest_stmt.c.icon,
latest_stmt.c.version_display,
# ownership
latest_stmt.c.owner_email,
# tags
latest_stmt.c.classifiers,
latest_stmt.c.quality,
# lifetime
latest_stmt.c.created,
latest_stmt.c.modified,
latest_stmt.c.deprecated,
).order_by(latest_stmt.c.key)


def can_get_service_stmt(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
can_get_service_stmt,
get_service_history_stmt,
get_service_stmt,
list_latest_services_with_history_stmt,
list_latest_services_stmt,
list_services_stmt,
total_count_stmt,
)
Expand Down Expand Up @@ -126,7 +126,7 @@ async def list_service_releases(
search_condition &= services_meta_data.c.version.like(f"{major}.%")

query = (
sa.select(SERVICES_META_DATA_COLS)
sa.select(*SERVICES_META_DATA_COLS)
.where(search_condition)
.order_by(sa.desc(services_meta_data.c.version))
)
Expand Down Expand Up @@ -386,15 +386,15 @@ async def list_latest_services(
user_id=user_id,
access_rights=AccessRightsClauses.can_read,
)
stmt_page = list_latest_services_with_history_stmt(
stmt_page = list_latest_services_stmt(
product_name=product_name,
user_id=user_id,
access_rights=AccessRightsClauses.can_read,
limit=limit,
offset=offset,
)

async with self.db_engine.begin() as conn:
async with self.db_engine.connect() as conn:
result = await conn.execute(stmt_total)
total_count = result.scalar() or 0

Expand Down Expand Up @@ -424,7 +424,7 @@ async def list_latest_services(
modified=r.modified,
deprecated=r.deprecated,
# releases
history=r.history,
history=[], # NOTE: for listing we will not add history. Only get service will produce history
)
for r in rows
]
Expand All @@ -446,7 +446,7 @@ async def get_service_history(
access_rights=AccessRightsClauses.can_read,
service_key=key,
)
async with self.db_engine.begin() as conn:
async with self.db_engine.connect() as conn:
result = await conn.execute(stmt_history)
row = result.one_or_none()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ def _as_dict(model_instance: ServiceMetaDataPublished) -> dict[str, Any]:
def get_function_service(key, version) -> ServiceMetaDataPublished:
try:
return next(
s
for s in iter_service_docker_data()
if s.key == key and s.version == version
sc
for sc in iter_service_docker_data()
if sc.key == key and sc.version == version
)
except StopIteration as err:
raise HTTPException(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
""" Services Manifest API Documentation
"""Services Manifest API Documentation

The `services.manifest` module provides a read-only API to access the services catalog. The term "Manifest" refers to a detailed, finalized list,
traditionally used to denote items that are recorded as part of an official inventory or log, emphasizing the immutable nature of the data.
Expand Down Expand Up @@ -60,7 +60,7 @@ async def get_services_map(

# NOTE: functional-services are services w/o associated image
services: ServiceMetaDataPublishedDict = {
(s.key, s.version): s for s in iter_service_docker_data()
(sc.key, sc.version): sc for sc in iter_service_docker_data()
}
for service in services_in_registry:
try:
Expand Down
Loading
Loading