Merge pull request #61 from machow/fix-rsconnect-full

Fix rsconnect full

Author: machow

MANIFEST.in

@@ -0,0 +1 @@
+include pins/data/*.csv

pins/boards.py

@@ -416,7 +416,7 @@ def pin_search(self, search=None, as_df=True):
             import pandas as pd
 
             # TODO(question): was the pulling of specific fields out a v0 thing?
-            extracted = list(map(self._extract_meta_results, res))
+            extracted = list(map(self._extract_search_meta, res))
             return pd.DataFrame(extracted)
 
         # TODO(compat): double check on the as_df=True convention
@@ -540,7 +540,7 @@ def prepare_pin_version(
     def _extract_search_meta(self, meta):
         keep_fields = ["name", "type", "title", "created", "file_size"]
 
-        d = {k: getattr(meta, k) for k in keep_fields}
+        d = {k: getattr(meta, k, None) for k in keep_fields}
         d["meta"] = meta
         return d
 
@@ -593,7 +593,7 @@ def pin_meta(self, name, version=None):
         path_to_pin = self.construct_path([pin_name])
         if self.fs.protocol == "http" and not path_to_pin.rstrip().endswith("/"):
             # create metadata, rather than read from a file
-            return self.meta_factory.create_raw(path_to_pin, type="file",)
+            return self.meta_factory.create_raw(path_to_pin, type="file", name=pin_name)
 
         path_meta = self.construct_path([pin_name, meta_name])
         f = self.fs.open(path_meta)
@@ -661,10 +661,8 @@ def pin_search(self, search=None, as_df=True):
                 # verify code is for inadequate permission to access
                 if e.args[0]["code"] != 19:
                     raise e
-                # TODO(question): should this be a MetaRaw class or something?
-                #                 that fixes our isinstance Meta below.
                 # TODO(compatibility): R pins errors instead, see #27
-                res.append({"name": pin_name, "meta": None})
+                res.append(self.meta_factory.create_raw(None, type=None, name=pin_name))
 
         # extract specific fields out ----
 
@@ -674,11 +672,7 @@ def pin_search(self, search=None, as_df=True):
 
             extract = []
             for entry in res:
-                extract.append(
-                    self._extract_search_meta(entry)
-                    if isinstance(entry, Meta)
-                    else entry
-                )
+                extract.append(self._extract_search_meta(entry))
 
             return pd.DataFrame(extract)
 
@@ -736,13 +730,15 @@ def user_name(self):
         user = self.fs.api.get_user()
         return user["username"]
 
-    def prepare_pin_version(self, pin_dir_path, x, *args, **kwargs):
-        from jinja2 import Environment
+    def prepare_pin_version(self, pin_dir_path, x, name: "str | None", *args, **kwargs):
 
-        env = Environment()
-        template = env.from_string(self.html_template.read_text())
+        # RSC pin names can have form <user_name>/<name>, but this will try to
+        # create the object in a directory named <user_name>. So we grab just
+        # the <name> part.
+        if "/" in name:
+            name = name.split("/")[-1]
 
-        meta = super().prepare_pin_version(pin_dir_path, x, *args, **kwargs)
+        meta = super().prepare_pin_version(pin_dir_path, x, name, *args, **kwargs)
 
         # copy in files needed by index.html ----------------------------------
         crnt_files = set([meta.file] if isinstance(meta.file, str) else meta.file)
@@ -788,6 +784,13 @@ def prepare_pin_version(self, pin_dir_path, x, *args, **kwargs):
             # TODO(compat): set display none in index.html
             context["data_preview"] = json.dumps({})
 
+        # render html template ----
+
+        from jinja2 import Environment
+
+        env = Environment()
+        template = env.from_string(self.html_template.read_text())
+
         rendered = template.render(context)
         (Path(pin_dir_path) / "index.html").write_text(rendered)
 

pins/data/__init__.py

@@ -1 +1,17 @@
-from siuba.data import mtcars  # noqa
+from importlib_resources import files as _files
+
+sources = {
+    "mtcars": _files("pins") / "data/mtcars.csv",
+}
+
+
+def __dir__():
+    return list(sources)
+
+
+def __getattr__(k):
+    import pandas as pd
+
+    f_path = sources.get("mtcars")
+
+    return pd.read_csv(f_path)

pins/data/mtcars.csv

@@ -0,0 +1,33 @@
+mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb
+21,6,160,110,3.9,2.62,16.46,0,1,4,4
+21,6,160,110,3.9,2.875,17.02,0,1,4,4
+22.8,4,108,93,3.85,2.32,18.61,1,1,4,1
+21.4,6,258,110,3.08,3.215,19.44,1,0,3,1
+18.7,8,360,175,3.15,3.44,17.02,0,0,3,2
+18.1,6,225,105,2.76,3.46,20.22,1,0,3,1
+14.3,8,360,245,3.21,3.57,15.84,0,0,3,4
+24.4,4,146.7,62,3.69,3.19,20,1,0,4,2
+22.8,4,140.8,95,3.92,3.15,22.9,1,0,4,2
+19.2,6,167.6,123,3.92,3.44,18.3,1,0,4,4
+17.8,6,167.6,123,3.92,3.44,18.9,1,0,4,4
+16.4,8,275.8,180,3.07,4.07,17.4,0,0,3,3
+17.3,8,275.8,180,3.07,3.73,17.6,0,0,3,3
+15.2,8,275.8,180,3.07,3.78,18,0,0,3,3
+10.4,8,472,205,2.93,5.25,17.98,0,0,3,4
+10.4,8,460,215,3,5.424,17.82,0,0,3,4
+14.7,8,440,230,3.23,5.345,17.42,0,0,3,4
+32.4,4,78.7,66,4.08,2.2,19.47,1,1,4,1
+30.4,4,75.7,52,4.93,1.615,18.52,1,1,4,2
+33.9,4,71.1,65,4.22,1.835,19.9,1,1,4,1
+21.5,4,120.1,97,3.7,2.465,20.01,1,0,3,1
+15.5,8,318,150,2.76,3.52,16.87,0,0,3,2
+15.2,8,304,150,3.15,3.435,17.3,0,0,3,2
+13.3,8,350,245,3.73,3.84,15.41,0,0,3,4
+19.2,8,400,175,3.08,3.845,17.05,0,0,3,2
+27.3,4,79,66,4.08,1.935,18.9,1,1,4,1
+26,4,120.3,91,4.43,2.14,16.7,0,1,5,2
+30.4,4,95.1,113,3.77,1.513,16.9,1,1,5,2
+15.8,8,351,264,4.22,3.17,14.5,0,1,5,4
+19.7,6,145,175,3.62,2.77,15.5,0,1,5,6
+15,8,301,335,3.54,3.57,14.6,0,1,5,8
+21.4,4,121,109,4.11,2.78,18.6,1,1,4,2

pins/meta.py

@@ -1,3 +1,4 @@
+from typing import ClassVar
 from dataclasses import dataclass, asdict, field
 
 import yaml
@@ -24,8 +25,9 @@ class MetaRaw:
         The type of pin data stored. This is used to determine how to read / write it.
     """
 
-    file: Union[str, Sequence[str]]
+    file: "str | Sequence[str] | None"
     type: str
+    name: str
 
 
 @dataclass
@@ -57,7 +59,7 @@ class Meta:
 
     """
 
-    title: str
+    title: Optional[str]
     description: Optional[str]
 
     # TODO(defer): different from R pins, which has a local field
@@ -93,19 +95,58 @@ def to_pin_dict(self):
     @classmethod
     def from_pin_dict(cls, data, pin_name, version) -> "Meta":
 
-        # version_fields = {"created", "pin_hash"}
-
-        # #get items necessary for re-creating meta data
-        # meta_data = {k: v for k, v in data.items() if k not in version_fields}
-        # version = version_cls.from_meta_fields(data["created"], data["pin_hash"])
-        return cls(**data, name=pin_name, version=version)
+        # TODO: re-arrange Meta argument positions to reflect what's been
+        # learned about default arguments. e.g. title was not used at some
+        # point in api_version 1
+        extra = {"title": None} if "title" not in data else {}
+        return cls(**data, **extra, name=pin_name, version=version)
 
     def to_pin_yaml(self, f: Optional[IOBase] = None) -> "str | None":
         data = self.to_pin_dict()
 
         return yaml.dump(data, f)
 
 
+@dataclass
+class MetaV0:
+    file: Union[str, Sequence[str]]
+    type: str
+
+    description: str
+    name: str
+
+    version: VersionRaw
+
+    # holds raw data.txt contents
+    original_fields: dict = field(default_factory=dict)
+    user: dict = field(default_factory=dict, init=False)
+
+    title: ClassVar[None] = None
+    created: ClassVar[None] = None
+    pin_hash: ClassVar[None] = None
+    file_size: ClassVar[None] = None
+    api_version: ClassVar[None] = None
+
+    def to_dict(self):
+        return asdict(self)
+
+    @classmethod
+    def from_pin_dict(cls, data, pin_name, version) -> "MetaV0":
+        # could infer from dataclasses.fields(), but seems excessive.
+        req_fields = {"type", "description", "name"}
+
+        req_inputs = {k: v for k, v in data.items() if k in req_fields}
+        req_inputs["file"] = data["path"]
+
+        return cls(**req_inputs, name=pin_name, original_fields=data, version=version)
+
+    def to_pin_dict(self):
+        raise NotImplementedError("v0 pins metadata are read only.")
+
+    def to_pin_yaml(self, *args, **kwargs):
+        self.to_pin_dict()
+
+
 class MetaFactory:
     """Responsible for creating and loading (e.g. from yaml) of meta objects.
 
@@ -168,8 +209,8 @@ def create(
             version=version,
         )
 
-    def create_raw(self, files: Sequence[StrOrFile], type: str = "file",) -> MetaRaw:
-        return MetaRaw(files, type)
+    def create_raw(self, files: Sequence[StrOrFile], type: str, name: str) -> MetaRaw:
+        return MetaRaw(files, type, name)
 
     def read_pin_yaml(
         self, f: IOBase, pin_name: str, version: "str | VersionRaw"
@@ -181,4 +222,14 @@ def read_pin_yaml(
 
         data = yaml.safe_load(f)
 
-        return Meta.from_pin_dict(data, pin_name, version=version_obj)
+        api_version = data.get("api_version", 0)
+        if api_version >= 2:
+            raise NotImplementedError(
+                f"api_version {api_version} by this version of the pins library"
+            )
+        elif api_version == 0:
+            cls_meta = MetaV0
+        else:
+            cls_meta = Meta
+
+        return cls_meta.from_pin_dict(data, pin_name, version=version_obj)

pins/rsconnect/api.py

@@ -133,9 +133,15 @@ class RsConnectApi:
     api_key: "str | None"
     server_url: "str"
 
-    def __init__(self, server_url, api_key=None):
+    def __init__(
+        self,
+        server_url: "str | None",
+        api_key: "str | None" = None,
+        session: "requests.Session | None" = None,
+    ):
         self.server_url = server_url
         self.api_key = api_key
+        self.session = requests.Session() if session is None else session
 
     # utility functions -------------------------------------------------------
 
@@ -165,10 +171,16 @@ def _get_api_key(self):
         if self.api_key is not None:
             return self.api_key
 
-        return os.environ[RSC_API_KEY]
+        return os.environ.get(RSC_API_KEY)
 
     def _get_headers(self):
-        return {"Authorization": f"key {self._get_api_key()}"}
+        api_key = self._get_api_key()
+        rsc_xsrf = self.session.cookies.get("RSC-XSRF")
+
+        d_key = {"Authorization": f"key {api_key}"} if api_key is not None else {}
+        d_rsc = {"X-RSC-XSRF": rsc_xsrf} if rsc_xsrf is not None else {}
+
+        return {**d_key, **d_rsc}
 
     def _validate_json_response(self, data: "dict | list"):
         if isinstance(data, list):
@@ -209,7 +221,7 @@ def _raw_query(self, url, method="GET", return_request=False, **kwargs):
 
         headers = self._get_headers()
 
-        r = requests.request(method, url, headers=headers, **kwargs)
+        r = self.session.request(method, url, headers=headers, **kwargs)
 
         if return_request:
             return r
@@ -418,46 +430,28 @@ def misc_get_applications(
 
 
 # ported from github.com/rstudio/connectapi
+# TODO: could just move these methods into RsConnectApi?
 class _HackyConnect(RsConnectApi):
     """Handles logging in to connect, rather than using an API key.
 
     This class allows you to create users and generate API keys on a fresh
     RStudio Connect service.
     """
 
-    xsrf: "None | str"
-
-    def __init__(self, *args, **kwargs):
-        self.xsrf = None
-        super().__init__(*args, **kwargs)
-
-    def _get_headers(self):
-        return {"X-RSC-XSRF": self.xsrf}
-
     def login(self, user, password):
         res = self.query(
             "__login__",
             "POST",
             return_request=True,
             json={"username": user, "password": password},
         )
-        self.xsrf = res.cookies["RSC-XSRF"]
         return res
 
     def create_first_admin(self, user, password, email, keyname="first-key"):
-        # TODO(question): this is run in the R rsconnect, but it returns json
-        # error codes. tests run okay without it...
-        # self.query_v1(
-        #     "users", "POST", json=dict(username=user, password=password, email=email)
-        #
-        # )
+        self.login(user, password)
 
-        res = self.login(user, password)
+        self.query("me")
 
-        self.query("me", cookies=res.cookies)
-
-        api_key = self.query(
-            "keys", "POST", json=dict(name=keyname), cookies=res.cookies
-        )
+        api_key = self.query("keys", "POST", json=dict(name=keyname))
 
         return RsConnectApi(self.server_url, api_key=api_key["key"])

pins/rsconnect/fs.py

@@ -1,6 +1,8 @@
 from dataclasses import dataclass, asdict, field, fields
 from pathlib import Path
 
+from fsspec import AbstractFileSystem
+
 from typing import Sequence
 
 from .api import (
@@ -93,7 +95,7 @@ class BundleFilePath(BundlePath):
     file_name: str
 
 
-class RsConnectFs:
+class RsConnectFs(AbstractFileSystem):
     protocol: str = "rsc"
 
     def __init__(self, server_url, **kwargs):