Merge pull request #196 from c00kiemon5ter/refactor-internal-data-deprecate-hashing

Refactor internal data to deprecate hashing

- Use the hasher micro-service to restore previous behaviour
- Use satosa.internal over satosa.internal_data
- Use satosa.internal.InternalData over satosa.internal_data.InternalRequest, satosa.internal_data.InternalResponse and satosa.backends.saml2.SAMLInternalResponse

For InternalData members:
- Use subject_id over user_id or name_id
- Use subject_type over user_id_hash_type
- Use attributes over approved_attributes

Author: c00kiemon5ter

doc/README.md

@@ -43,7 +43,7 @@ in the [example directory](../example).
 | `BACKEND_MODULES` | string[] | `[openid_connect_backend.yaml, saml2_backend.yaml]` | list of plugin configuration file paths, describing enabled backends |
 | `FRONTEND_MODULES` | string[] | `[saml2_frontend.yaml, openid_connect_frontend.yaml]` | list of plugin configuration file paths, describing enabled frontends |
 | `MICRO_SERVICES` | string[] | `[statistics_service.yaml]` | list of plugin configuration file paths, describing enabled microservices |
-| `USER_ID_HASH_SALT` | string | `61a89d2db0b9e1e2` | salt used when creating the persistent user identifier, will be overriden by the environment variable `SATOSA_USER_ID_HASH_SALT` if it is set |
+| `USER_ID_HASH_SALT` | string | `61a89d2db0b9e1e2` | **DEPRECATED - use the hasher micro-service** salt used when creating the persistent user identifier, will be overriden by the environment variable `SATOSA_USER_ID_HASH_SALT` if it is set |
 | `LOGGING` | dict | see [Python logging.conf](https://docs.python.org/3/library/logging.config.html) | optional configuration of application logging |
 
 
@@ -119,7 +119,7 @@ linking, the `user_id_to_attr` configuration parameter should be set, since that
 service will overwrite the user identifier generated by the proxy.
 
 
-### hash
+### hash **DEPRECATED - use the hasher micro-service**
 The proxy can hash any attribute value (e.g., for obfuscation) before passing
 it on to the client. The `hash` key should contain a list of all attribute names
 for which the corresponding attribute values should be hashed before being
@@ -410,7 +410,7 @@ which should be used when configuring the attribute mapping (see above).
 ### Ping frontend for simple heartbeat monitoring
 
 The ping frontend responds to a query with a simple
-200 OK and is intended to be used as a simple heartbeat monitor, 
+200 OK and is intended to be used as a simple heartbeat monitor,
 for example by a load balancer. The default configuration file can
 be found [here](../example/plugins/frontends/ping_frontend.yaml.example).
 
@@ -581,7 +581,7 @@ The SATOSA proxy is a Python WSGI application and so may be run using any WSGI c
 
 Gunicorn 'Green Unicorn' is a Python WSGI HTTP Server for UNIX and is the server used most often
 to run the proxy. In a production deployment the Gunicorn server is often proxied by a
-full featured general purpose web server (in a reverse proxy architecture) such as Nginx or 
+full featured general purpose web server (in a reverse proxy architecture) such as Nginx or
 Apache HTTP Server to help buffer slow clients and enable more sophisticated error page rendering.
 
 Start the proxy server with the following command:

example/internal_attributes.yaml.example

@@ -40,6 +40,5 @@ attributes:
     orcid: [name.family-name.value]
     openid: [family_name]
     saml: [sn, surname]
-hash: [edupersontargetedid]
 user_id_from_attrs: [edupersontargetedid]
 user_id_to_attr: edupersontargetedid

example/plugins/microservices/hasher.yaml.example

@@ -0,0 +1,39 @@
+module: satosa.micro_services.hasher.Hasher
+name: Hasher
+config:
+  # default settings that apply to every requester
+  "":
+    # default salt configuration is required
+    salt: abcdef0123456789
+
+    # the hash algorithm to use (default: sha512)
+    alg: sha256
+
+    # whether subject_id should be hashed (default: yes)
+    subject_id: yes
+
+    # a list of attributes to hash (default: [])
+    attributes:
+    - edupersontargetedid
+
+  # specific settings for requester 'some_entityid'
+  some_entityid:
+    # for this requester use sha1
+    alg: sha1
+
+    # do not hash any attributes
+    # if this is missing the defaults will be used
+    attributes: []
+
+  # specific settings for requester 'some_other_entityid'
+  some_other_entityid:
+    # for this requester only use this salt
+    salt: abcd1234
+
+    # do not hash subject_id
+    subject_id: no
+
+    # only hash the following attributes
+    attributes:
+    - gender
+    - identifier

example/proxy_conf.yaml.example

@@ -13,7 +13,6 @@ FRONTEND_MODULES:
   - "plugins/frontends/saml2_frontend.yaml"
 MICRO_SERVICES:
   - "plugins/microservices/static_attributes.yaml"
-USER_ID_HASH_SALT: "61a89d2db0b9e1e27d490d050b478fe71f352fddd3528a44157f43e339c6c62f2362fb413179937d96172bf84233317"
 LOGGING:
   version: 1
   formatters:
@@ -40,4 +39,4 @@ LOGGING:
       propagate: no
   root:
     level: INFO
-    handlers: [info_file_handler]
+    handlers: [info_file_handler]

src/satosa/backends/base.py

@@ -13,7 +13,7 @@ class BackendModule(object):
     def __init__(self, auth_callback_func, internal_attributes, base_url, name):
         """
         :type auth_callback_func:
-        (satosa.context.Context, satosa.internal_data.InternalResponse) -> satosa.response.Response
+        (satosa.context.Context, satosa.internal.InternalData) -> satosa.response.Response
         :type internal_attributes: dict[string, dict[str, str | list[str]]]
         :type base_url: str
         :type name: str
@@ -37,7 +37,7 @@ def start_auth(self, context, internal_request):
         This is the start up function of the backend authorization.
 
         :type context: satosa.context.Context
-        :type internal_request: satosa.internal_data.InternalRequest
+        :type internal_request: satosa.internal.InternalData
         :rtype satosa.response.Response
 
         :param context: the request context

src/satosa/backends/github.py

@@ -10,10 +10,10 @@
 from oic.oauth2.message import AuthorizationResponse
 
 from satosa.backends.oauth import _OAuthBackend
-from ..internal_data import InternalResponse
-from ..internal_data import AuthenticationInformation
-from ..response import Redirect
-from ..util import rndstr
+from satosa.internal import AuthenticationInformation
+from satosa.internal import InternalData
+from satosa.response import Redirect
+from satosa.util import rndstr
 
 logger = logging.getLogger(__name__)
 
@@ -32,7 +32,7 @@ def __init__(self, outgoing, internal_attributes, config, base_url, name):
         :param base_url: base url of the service
         :param name: name of the plugin
         :type outgoing:
-            (satosa.context.Context, satosa.internal_data.InternalResponse) ->
+            (satosa.context.Context, satosa.internal.InternalData) ->
             satosa.response.Response
         :type internal_attributes: dict[string, dict[str, str | list[str]]]
         :type config: dict[str, dict[str, str] | list[str] | str]
@@ -51,7 +51,7 @@ def start_auth(self, context, internal_request, get_state=stateID):
 
         :type get_state: Callable[[str, bytes], str]
         :type context: satosa.context.Context
-        :type internal_request: satosa.internal_data.InternalRequest
+        :type internal_request: satosa.internal.InternalData
         :rtype satosa.response.Redirect
         """
         oauth_state = get_state(self.config["base_url"], rndstr().encode())
@@ -95,10 +95,10 @@ def _authn_response(self, context):
 
         user_info = self.user_information(response["access_token"])
         auth_info = self.auth_info(context.request)
-        internal_response = InternalResponse(auth_info=auth_info)
+        internal_response = InternalData(auth_info=auth_info)
         internal_response.attributes = self.converter.to_internal(
             self.external_type, user_info)
-        internal_response.user_id = str(user_info[self.user_id_attr])
+        internal_response.subject_id = str(user_info[self.user_id_attr])
         del context.state[self.name]
         return self.auth_callback_func(context, internal_response)
 

src/satosa/backends/linkedin.py

@@ -10,10 +10,10 @@
 from oic.oauth2.message import AuthorizationResponse
 
 from satosa.backends.oauth import _OAuthBackend
-from ..internal_data import InternalResponse
-from ..internal_data import AuthenticationInformation
-from ..response import Redirect
-from ..util import rndstr
+from satosa.internal import AuthenticationInformation
+from satosa.internal import InternalData
+from satosa.response import Redirect
+from satosa.util import rndstr
 
 
 logger = logging.getLogger(__name__)
@@ -33,7 +33,7 @@ def __init__(self, outgoing, internal_attributes, config, base_url, name):
         :param base_url: base url of the service
         :param name: name of the plugin
         :type outgoing:
-            (satosa.context.Context, satosa.internal_data.InternalResponse) ->
+            (satosa.context.Context, satosa.internal.InternalData) ->
             satosa.response.Response
         :type internal_attributes: dict[string, dict[str, str | list[str]]]
         :type config: dict[str, dict[str, str] | list[str] | str]
@@ -52,7 +52,7 @@ def start_auth(self, context, internal_request, get_state=stateID):
 
         :type get_state: Callable[[str, bytes], str]
         :type context: satosa.context.Context
-        :type internal_request: satosa.internal_data.InternalRequest
+        :type internal_request: satosa.internal.InternalData
         :rtype satosa.response.Redirect
         """
         oauth_state = get_state(self.config["base_url"], rndstr().encode())
@@ -96,10 +96,10 @@ def _authn_response(self, context):
 
         user_info = self.user_information(response["access_token"])
         auth_info = self.auth_info(context.request)
-        internal_response = InternalResponse(auth_info=auth_info)
+        internal_response = InternalData(auth_info=auth_info)
         internal_response.attributes = self.converter.to_internal(
             self.external_type, user_info)
-        internal_response.user_id = user_info[self.user_id_attr]
+        internal_response.subject_id = user_info[self.user_id_attr]
         del context.state[self.name]
         return self.auth_callback_func(context, internal_response)
 

src/satosa/backends/oauth.py

@@ -10,13 +10,17 @@
 from oic.oauth2.message import AuthorizationResponse
 from oic.utils.authn.authn_context import UNSPECIFIED
 
-from .base import BackendModule
-from ..exception import SATOSAAuthenticationError
-from ..internal_data import InternalResponse, AuthenticationInformation
-from ..logging_util import satosa_logging
-from ..metadata_creation.description import OrganizationDesc, UIInfoDesc, ContactPersonDesc, MetadataDescription
-from ..response import Redirect
-from ..util import rndstr
+from satosa.internal import AuthenticationInformation
+from satosa.internal import InternalData
+from satosa.exception import SATOSAAuthenticationError
+from satosa.logging_util import satosa_logging
+from satosa.response import Redirect
+from satosa.util import rndstr
+from satosa.metadata_creation.description import (
+    OrganizationDesc, UIInfoDesc, ContactPersonDesc, MetadataDescription
+)
+from satosa.backends.base import BackendModule
+
 
 logger = logging.getLogger(__name__)
 
@@ -40,7 +44,7 @@ def __init__(self, outgoing, internal_attributes, config, base_url, name, extern
         :param external_type: The name for this module in the internal attributes.
 
         :type outgoing:
-        (satosa.context.Context, satosa.internal_data.InternalResponse) -> satosa.response.Response
+        (satosa.context.Context, satosa.internal.InternalData) -> satosa.response.Response
         :type internal_attributes: dict[string, dict[str, str | list[str]]]
         :type config: dict[str, dict[str, str] | list[str]]
         :type base_url: str
@@ -67,7 +71,7 @@ def start_auth(self, context, internal_request, get_state=stateID):
 
         :type get_state: Callable[[str, bytes], str]
         :type context: satosa.context.Context
-        :type internal_request: satosa.internal_data.InternalRequest
+        :type internal_request: satosa.internal.InternalData
         :rtype satosa.response.Redirect
         """
         oauth_state = get_state(self.config["base_url"], rndstr().encode())
@@ -132,9 +136,9 @@ def _authn_response(self, context):
             self._verify_state(atresp, state_data, context.state)
 
         user_info = self.user_information(atresp["access_token"])
-        internal_response = InternalResponse(auth_info=self.auth_info(context.request))
+        internal_response = InternalData(auth_info=self.auth_info(context.request))
         internal_response.attributes = self.converter.to_internal(self.external_type, user_info)
-        internal_response.user_id = user_info[self.user_id_attr]
+        internal_response.subject_id = user_info[self.user_id_attr]
         del context.state[self.name]
         return self.auth_callback_func(context, internal_response)
 
@@ -187,7 +191,7 @@ def __init__(self, outgoing, internal_attributes, config, base_url, name):
         :param name: name of the plugin
 
         :type outgoing:
-        (satosa.context.Context, satosa.internal_data.InternalResponse) -> satosa.response.Response
+        (satosa.context.Context, satosa.internal.InternalData) -> satosa.response.Response
         :type internal_attributes: dict[string, dict[str, str | list[str]]]
         :type config: dict[str, dict[str, str] | list[str] | str]
         :type base_url: str

src/satosa/backends/openid_connect.py

@@ -12,10 +12,11 @@
 from oic.utils.authn.authn_context import UNSPECIFIED
 from oic.utils.authn.client import CLIENT_AUTHN_METHOD
 
+from satosa.internal import AuthenticationInformation
+from satosa.internal import InternalData
 from .base import BackendModule
 from .oauth import get_metadata_desc_for_oauth_backend
 from ..exception import SATOSAAuthenticationError, SATOSAError
-from ..internal_data import InternalResponse, AuthenticationInformation
 from ..logging_util import satosa_logging
 from ..response import Redirect
 
@@ -43,7 +44,7 @@ def __init__(self, auth_callback_func, internal_attributes, config, base_url, na
         :param name: name of the plugin
 
         :type auth_callback_func:
-        (satosa.context.Context, satosa.internal_data.InternalResponse) -> satosa.response.Response
+        (satosa.context.Context, satosa.internal.InternalData) -> satosa.response.Response
         :type internal_attributes: dict[string, dict[str, str | list[str]]]
         :type config: dict[str, dict[str, str] | list[str]]
         :type base_url: str
@@ -66,7 +67,7 @@ def start_auth(self, context, request_info):
         """
         See super class method satosa.backends.base#start_auth
         :type context: satosa.context.Context
-        :type request_info: satosa.internal_data.InternalRequest
+        :type request_info: satosa.internal.InternalData
         """
         oidc_nonce = rndstr()
         oidc_state = rndstr()
@@ -213,17 +214,17 @@ def _translate_response(self, response, issuer):
         :type response: dict[str, str]
         :type issuer: str
         :type subject_type: str
-        :rtype: InternalResponse
+        :rtype: InternalData
 
         :param response: Dictioary with attribute name as key.
         :param issuer: The oidc op that gave the repsonse.
         :param subject_type: public or pairwise according to oidc standard.
         :return: A SATOSA internal response.
         """
         auth_info = AuthenticationInformation(UNSPECIFIED, str(datetime.now()), issuer)
-        internal_resp = InternalResponse(auth_info=auth_info)
+        internal_resp = InternalData(auth_info=auth_info)
         internal_resp.attributes = self.converter.to_internal("openid", response)
-        internal_resp.user_id = response["sub"]
+        internal_resp.subject_id = response["sub"]
         return internal_resp
 
     def get_metadata_desc(self):

src/satosa/backends/orcid.py

@@ -11,9 +11,9 @@
 from oic.oauth2.message import AuthorizationResponse
 
 from satosa.backends.oauth import _OAuthBackend
-from ..internal_data import InternalResponse
-from ..internal_data import AuthenticationInformation
-from ..response import Redirect
+from satosa.internal import InternalData
+from satosa.internal import AuthenticationInformation
+from satosa.response import Redirect
 
 logger = logging.getLogger(__name__)
 
@@ -32,7 +32,7 @@ def __init__(self, outgoing, internal_attributes, config, base_url, name):
         :param base_url: base url of the service
         :param name: name of the plugin
         :type outgoing:
-            (satosa.context.Context, satosa.internal_data.InternalResponse) ->
+            (satosa.context.Context, satosa.internal.InternalData) ->
             satosa.response.Response
         :type internal_attributes: dict[string, dict[str, str | list[str]]]
         :type config: dict[str, dict[str, str] | list[str] | str]
@@ -51,7 +51,7 @@ def start_auth(self, context, internal_request, get_state=stateID):
 
         :type get_state: Callable[[str, bytes], str]
         :type context: satosa.context.Context
-        :type internal_request: satosa.internal_data.InternalRequest
+        :type internal_request: satosa.internal.InternalData
         :rtype satosa.response.Redirect
         """
         request_args = dict(
@@ -85,10 +85,10 @@ def _authn_response(self, context):
         orcid, name = response['orcid'], response['name']
         user_info = self.user_information(token, orcid, name)
         auth_info = self.auth_info(context.request)
-        internal_response = InternalResponse(auth_info=auth_info)
+        internal_response = InternalData(auth_info=auth_info)
         internal_response.attributes = self.converter.to_internal(
             self.external_type, user_info)
-        internal_response.user_id = orcid
+        internal_response.subject_id = orcid
         return self.auth_callback_func(context, internal_response)
 
     def user_information(self, access_token, orcid, name):