'v7.x alpha' rebased onto 'master minus black' (#260)

Update python-pachyderm for Pachyderm 2.0

Author: msteffen

.travis.yml

@@ -15,21 +15,19 @@ env:
   global:
     - PATH=$HOME/cached-deps:$PATH
   jobs:
-    # Most recent releases of the last two minor releases of Pachyderm
-    - PACHYDERM_VERSION=1.12.5
-    - PACHYDERM_VERSION=1.13.0
+    - PACHYDERM_VERSION=2.0.0-alpha.12
 
 jobs:
   include:
     # other tests
     - python: 3.8
       env:
         - TOXENV=lint
-        - PACHYDERM_VERSION=1.13.0
+        - PACHYDERM_VERSION=2.0.0-alpha.12
     - python: 3.8
       env:
         - TOXENV=examples
-        - PACHYDERM_VERSION=1.13.0
+        - PACHYDERM_VERSION=2.0.0-alpha.12
 install:
 - make ci-install
 before_script:

Makefile

@@ -23,12 +23,12 @@ src/python_pachyderm/proto: docker-build-proto
 	cd proto/pachyderm && \
 		git fetch --all && \
 		git checkout v$(PACHYDERM_VERSION)
-	find ./proto/pachyderm/src/client -regex ".*\.proto" \
+	find ./proto/pachyderm/ -regex ".*\.proto" \
 	| grep -v 'internal' \
 	| xargs tar cf - \
 	| docker run -i pachyderm_python_proto \
 	| tar xf -
-	mv src/python_pachyderm/client src/python_pachyderm/proto
+	mv src/python_pachyderm/src src/python_pachyderm/proto
 	find src/python_pachyderm/proto -type d -exec touch {}/__init__.py \;
 
 init:
@@ -61,4 +61,4 @@ lint:
 	flake8 src/python_pachyderm --exclude=src/python_pachyderm/proto --max-line-length=120 --max-doc-length=80
 	PYTHONPATH=./src:$(PYTHONPATH) etc/proto_lint/proto_lint.py
 
-.PHONY: docker-build-proto init ci-install ci-setup release lint
+.PHONY: docs docker-build-proto init ci-install ci-setup release lint

docs/python_pachyderm.html

@@ -23,6 +23,8 @@
     Service.DEBUG: mixin.debug.DebugMixin,
     Service.ENTERPRISE: mixin.enterprise.EnterpriseMixin,
     Service.HEALTH: mixin.health.HealthMixin,
+    Service.IDENTITY: mixin.identity.IdentityMixin,
+    Service.LICENSE: mixin.license.LicenseMixin,
     Service.PFS: mixin.pfs.PFSMixin,
     Service.PPS: mixin.pps.PPSMixin,
     Service.TRANSACTION: mixin.transaction.TransactionMixin,
@@ -66,23 +68,34 @@
         "build_commit",
         "put_tar",
         "get_tar",
-        # ignore storage v2 for now
-        "get_tar_conditional_v2",
-        "file_operation_v2",
-        "list_file_v2",
-        "glob_file_v2",
-        "get_tar_v2",
-        "walk_file_v2",
-        "inspect_file_v2",
-        "clear_commit_v2",
-        "diff_file_v2",
+        "activate_auth",
+        # TODO: add these new API methods
+        "renew_fileset",
+        "squash_commit",
+        "clear_commit",
+        "modify_file",
+        "add_fileset",
+        "get_fileset",
+        "create_fileset",
     ],
     Service.PPS: [
         # the following are ignored because they're for internal use only
         "activate_auth",
         "create_job",
         "update_job_state",
     ],
+    Service.AUTH: [
+        "get_auth_token",
+        "extend_auth_token",
+    ],
+    Service.ENTERPRISE: [
+        # internal RPC only
+        "heartbeat",
+    ],
+    Service.LICENSE: [
+        # internal RPC only
+        "heartbeat",
+    ],
 }
 
 # Mapping of what the linter would by default expect a proto name to be, to
@@ -91,7 +104,7 @@
     Service.AUTH: {
         "activate": ["activate_auth"],
         "deactivate": ["deactivate_auth"],
-        "authenticate": ["authenticate_github", "authenticate_oidc", "authenticate_one_time_password"],
+        "authenticate": ["authenticate_oidc"],
         "get_a_c_l": ["get_acl"],
         "set_a_c_l": ["set_acl"],
         "get_o_i_d_c_login": ["get_oidc_login"],
@@ -106,6 +119,23 @@
         "get_state": ["get_enterprise_state"],
         "deactivate": ["deactivate_enterprise"],
     },
+    Service.LICENSE: {
+        "activate": ["activate_license"],
+        "delete_all": ["delete_all_license"],
+    },
+    Service.IDENTITY: {
+        "get_i_d_p_connector": ["get_idp_connector"],
+        "delete_i_d_p_connector": ["delete_idp_connector"],
+        "list_i_d_p_connectors": ["list_idp_connectors"],
+        "create_i_d_p_connector": ["create_idp_connector"],
+        "update_i_d_p_connector": ["update_idp_connector"],
+	"get_o_i_d_c_client": ["get_oidc_client"],
+        "delete_o_i_d_c_client": ["delete_oidc_client"],
+        "list_o_i_d_c_clients": ["list_oidc_clients"],
+        "create_o_i_d_c_client": ["create_oidc_client"],
+        "update_o_i_d_c_client": ["update_oidc_client"],
+        "delete_all": ["delete_all_identity"],
+    },
     Service.PFS: {
         "put_file": ["put_file_bytes", "put_file_url"],
     },
@@ -132,27 +162,7 @@
 # * We can also specify that a method has an argument that isn't in the
 #   protos: `(None, "ignored_arg_name")`
 RENAMED_ARGS = {
-    # admin
-    "extract_pipeline": [
-        ("pipeline", "pipeline_name"),
-    ],
-    "extract": [
-        ("URL", "url"),
-    ],
-    "restore": [
-        (("op", "URL"), "requests"),
-    ],
     # auth
-    "authenticate_github": [
-        ("one_time_password", None),
-        ("oidc_state", None),
-        ("id_token", None),
-    ],
-    "authenticate_one_time_password": [
-        ("github_token", None),
-        ("oidc_state", None),
-        ("id_token", None),
-    ],
     "authenticate_oidc": [
         ("github_token", None),
         ("one_time_password", None),

etc/proto_lint/proto_lint.py

@@ -1,26 +1,31 @@
-#!/bin/sh
+#!/bin/sh -x 
 
 tar xf /dev/stdin
 
 OUTDIR=src/python_pachyderm/
 mkdir -p $OUTDIR
 
-for i in $(find ./proto/pachyderm/src/client -name "*.proto"); do \
+for i in $(find ./proto/pachyderm/src -name "*.proto"); do \
     # Make sure to remove the gogoproto line, as that is go specific
     sed -i s/import.*gogo.proto.*\;// $i
     sed -i 's/\[.*gogoproto.*\]//' $i
     # We need to do this in a pass before compilation because it's not guaranteed we
     # traverse the proto files in order of dependence
 done
 
-# skip pkg because protos in there aren't part of the public API
-for i in $(find ./proto/pachyderm/src/client -name "*.proto" | grep -v "proto/pachyderm/src/client/pkg"); do \
-    python3 -m grpc_tools.protoc -I./proto/pachyderm/src --python_out=$OUTDIR --grpc_python_out=$OUTDIR ${i}; \
+# fix imports to be relative to src/
+for i in $(find ./proto/pachyderm/src -name "*.proto" | grep -v "proto/pachyderm/src/internal"); do \
+    perl -pi.bak -e 's/import "((?!google).*)"/import "src\/$1"/' $i
+done
+
+# skip pkg and server because protos in there aren't part of the public API
+for i in $(find ./proto/pachyderm/src -name "*.proto" | grep -v "proto/pachyderm/src/internal" | grep -v 'proto/pachyderm/src/server'); do \
+    python3 -m grpc_tools.protoc -I./proto/pachyderm --python_out=$OUTDIR --grpc_python_out=$OUTDIR ${i}; \
 done
 
 for i in $(find $OUTDIR -regex ".*pb2.*py"); do
     # Fix the imports so they work once this is packaged by pypi
-    sed -i 's/from client\./from python_pachyderm\.proto\./' $i
+    sed -i 's/from src\./from python_pachyderm\.proto\./' $i
 done
 
 

proto/pachyderm

@@ -3,7 +3,7 @@
 import enum as _enum
 from google.protobuf.internal.enum_type_wrapper import EnumTypeWrapper as _EnumTypeWrapper
 
-from .mixin.pfs import PFSFile, PutFileClient
+from .mixin.pfs import PFSFile, ModifyFileClient
 from .client import Client, ConfigError, BadClusterDeploymentID
 from .spout import SpoutManager, SpoutCommit
 from .util import put_files, create_python_pipeline, parse_json_pipeline_spec, parse_dict_pipeline_spec
@@ -18,7 +18,7 @@
     "put_files",
     "create_python_pipeline",
     "PFSFile",
-    "PutFileClient",
+    "ModifyFileClient",
     "parse_json_pipeline_spec",
     "parse_dict_pipeline_spec",
     "ConfigError",
@@ -88,3 +88,5 @@ def should_import(key):
 _import_protos("python_pachyderm.proto.admin.admin_pb2")
 _import_protos("python_pachyderm.proto.auth.auth_pb2")
 _import_protos("python_pachyderm.proto.enterprise.enterprise_pb2")
+_import_protos("python_pachyderm.proto.license.license_pb2")
+_import_protos("python_pachyderm.proto.identity.identity_pb2")

proto/run

@@ -8,6 +8,8 @@
 from .mixin.debug import DebugMixin
 from .mixin.enterprise import EnterpriseMixin
 from .mixin.health import HealthMixin
+from .mixin.identity import IdentityMixin
+from .mixin.license import LicenseMixin
 from .mixin.pfs import PFSMixin
 from .mixin.pps import PPSMixin
 from .mixin.transaction import TransactionMixin
@@ -40,6 +42,8 @@ class Client(
     DebugMixin,
     EnterpriseMixin,
     HealthMixin,
+    IdentityMixin,
+    LicenseMixin,
     PFSMixin,
     PPSMixin,
     TransactionMixin,

src/python_pachyderm/__init__.py

@@ -1,59 +1,8 @@
 from python_pachyderm.proto.admin import admin_pb2_grpc as admin_grpc
-from python_pachyderm.proto.pps import pps_pb2 as pps_proto
 from python_pachyderm.service import Service
 
 
 class AdminMixin:
-    def extract(self,
-                url=None, no_objects=None, no_repos=None, no_pipelines=None,
-                no_auth=None, no_enterprise=None):
-        """
-        Extracts cluster data for backup. Yields `Op` objects.
-
-        Params:
-
-        * `url`: An optional string specifying an object storage URL. If set,
-          data will be extracted to this URL rather than returned.
-        * `no_objects`: An optional bool. If true, will cause extract to omit
-          objects (and tags.)
-        * `no_repos`: An optional bool. If true, will cause extract to omit
-          repos, commits and branches.
-        * `no_pipelines`: An optional bool. If true, will cause extract to
-          omit pipelines.
-        * `no_auth`: An optional bool. If true, will cause extract to omit
-          acls, tokens, etc.
-        * `no_enterprise`: An optional bool. If true, will cause extract to
-          omit any enterprise activation key (which may break auth restore)
-        """
-        return self._req(
-            Service.ADMIN, "Extract",
-            URL=url or "",
-            no_objects=no_objects, no_repos=no_repos, no_pipelines=no_pipelines,
-            no_auth=no_auth, no_enterprise=no_enterprise,
-        )
-
-    def extract_pipeline(self, pipeline_name):
-        """
-        Extracts a pipeline for backup. Returns an `Op` object.
-
-        Params:
-
-        * `pipeline_name`: A string representing a pipeline name to extract.
-        """
-        return self._req(
-            Service.ADMIN, "ExtractPipeline",
-            pipeline=pps_proto.Pipeline(name=pipeline_name) if pipeline_name is not None else None,
-        )
-
-    def restore(self, requests):
-        """
-        Restores a cluster.
-
-        Params:
-
-        * `requests`: A generator of `RestoreRequest` objects.
-        """
-        return self._req(Service.ADMIN, "Restore", req=requests)
 
     def inspect_cluster(self):
         """