Merge pull request #178 from rstudio/feat-manifest-compat

Feat manifest compat

Author: machow

pins/boards.py

@@ -53,6 +53,8 @@ def info(self, path):
 
 
 class BaseBoard:
+    reserved_pin_names = {"_pins.yaml"}
+
     def __init__(
         self,
         board: "str | Path",
@@ -174,8 +176,11 @@ def pin_list(self):
         This is a low-level function; use pin_search() to get more data about
         each pin in a convenient form.
         """
+
         full_paths = self.fs.ls(self.board)
-        return list(map(self.keep_final_path_component, full_paths))
+        pin_names = map(self.keep_final_path_component, full_paths)
+
+        return [name for name in pin_names if name not in self.reserved_pin_names]
 
     def pin_fetch(self, name: str, version: Optional[str] = None) -> Meta:
         meta = self.pin_meta(name, version)
@@ -513,6 +518,8 @@ def validate_pin_name(self, name: str) -> None:
 
         if "/" in name:
             raise ValueError(f"Invalid pin name: {name}")
+        elif name in self.reserved_pin_names:
+            raise ValueError(f"The pin name '{name}' is reserved for internal use.")
 
     def path_to_pin(self, name: str) -> str:
         self.validate_pin_name(name)

pins/meta.py

@@ -1,10 +1,10 @@
 from typing import ClassVar
-from dataclasses import dataclass, asdict, field
+from dataclasses import dataclass, asdict, field, fields, InitVar
 from pathlib import Path
 
 import yaml
 
-from typing import Mapping, Union, Sequence, Optional
+from typing import Mapping, Union, Sequence, Optional, List
 
 from .versions import VersionRaw, Version, guess_version
 from ._types import StrOrFile, IOBase
@@ -41,6 +41,8 @@ class Meta:
         A title for the pin.
     description:
         A detailed description of the pin contents.
+    tags:
+        Optional tags applied to the pin.
     created:
         Datetime the pin was created (TODO: document format).
     pin_hash:
@@ -63,6 +65,8 @@ class Meta:
 
     """
 
+    _excluded: ClassVar["set[str]"] = {"name", "version", "local"}
+
     title: Optional[str]
     description: Optional[str]
 
@@ -82,10 +86,24 @@ class Meta:
     # pin_hash, created, etc.."
     version: VersionRaw
 
+    tags: Optional[List[str]] = None
     name: Optional[str] = None
     user: Mapping = field(default_factory=dict)
     local: Mapping = field(default_factory=dict)
 
+    unknown_fields: InitVar["dict | None"] = None
+
+    def __post_init__(self, unknown_fields: "dict | None"):
+        unknown_fields = {} if unknown_fields is None else unknown_fields
+
+        self._unknown_fields = unknown_fields
+
+    def __getattr__(self, k):
+        try:
+            return self._unknown_fields[k]
+        except KeyError:
+            raise AttributeError(f"No metadata field not found: {k}")
+
     def to_dict(self) -> Mapping:
         data = asdict(self)
 
@@ -94,21 +112,37 @@ def to_dict(self) -> Mapping:
     def to_pin_dict(self):
         d = self.to_dict()
 
-        del d["name"]
-        del d["version"]
-        del d["local"]
+        for k in self._excluded:
+            del d[k]
+
+        # TODO: once tag writing is implemented, delete this line
+        del d["tags"]
 
         return d
 
     @classmethod
     def from_pin_dict(cls, data, pin_name, version, local=None) -> "Meta":
-
         # TODO: re-arrange Meta argument positions to reflect what's been
         # learned about default arguments. e.g. title was not used at some
         # point in api_version 1
+        all_field_names = {entry.name for entry in fields(Meta)}
+
+        keep_fields = all_field_names - cls._excluded
+
         extra = {"title": None} if "title" not in data else {}
         local = {} if local is None else local
-        return cls(**data, **extra, name=pin_name, version=version, local=local)
+
+        meta_data = {k: v for k, v in data.items() if k in keep_fields}
+        unknown = {k: v for k, v in data.items() if k not in keep_fields}
+
+        return cls(
+            **meta_data,
+            **extra,
+            name=pin_name,
+            version=version,
+            local=local,
+            unknown_fields=unknown,
+        )
 
     def to_pin_yaml(self, f: Optional[IOBase] = None) -> "str | None":
         data = self.to_pin_dict()

pins/tests/conftest.py

@@ -12,6 +12,7 @@
 PATH_TO_EXAMPLE_VERSION = PATH_TO_EXAMPLE_BOARD / "df_csv/20220214T163720Z-9bfad/"
 EXAMPLE_PIN_NAME = "df_csv"
 
+PATH_TO_MANIFEST_BOARD = files("pins") / "tests/pin-board"
 
 # Based on https://github.com/machow/siuba/blob/main/siuba/tests/helpers.py
 BACKEND_MARKS = ["fs_s3", "fs_file", "fs_gcs", "fs_abfs", "fs_rsc"]

pins/tests/pin-board/_pins.yaml

@@ -0,0 +1,5 @@
+x:
+- x/20221215T180351Z-c3943/
+'y':
+- y/20221215T180357Z-9ae7a/
+- y/20221215T180400Z-b81d5/

pins/tests/pin-board/x/20221215T180351Z-c3943/data.txt

@@ -0,0 +1,9 @@
+file: x.json
+file_size: 23
+pin_hash: c3943ca5a9aab2df
+type: json
+title: 'x: a pinned integer vector'
+description: ~
+tags: ~
+created: 20221215T180351Z
+api_version: 1.0

pins/tests/pin-board/x/20221215T180351Z-c3943/x.json

@@ -0,0 +1 @@
+[1,2,3,4,5,6,7,8,9,10]

pins/tests/pin-board/y/20221215T180357Z-9ae7a/data.txt

@@ -0,0 +1,9 @@
+file: y.rds
+file_size: 61
+pin_hash: 9ae7a970010c84e0
+type: rds
+title: 'y: a pinned integer vector'
+description: ~
+tags: ~
+created: 20221215T180357Z
+api_version: 1.0

pins/tests/pin-board/y/20221215T180357Z-9ae7a/y.rds

@@ -0,0 +1,9 @@
+file: y.json
+file_size: 53
+pin_hash: b81d5bea9e760608
+type: json
+title: 'y: a pinned integer vector'
+description: ~
+tags: ~
+created: 20221215T180400Z
+api_version: 1.0

pins/tests/pin-board/y/20221215T180400Z-b81d5/data.txt

@@ -0,0 +1 @@
+[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]