So not completely clear on what the standard says. Had to add this to make it work with Shibboleth IdP.

Author: Roland Hedberg

src/saml2/entity.py

@@ -1,4 +1,5 @@
 import base64
+from binascii import hexlify
 import logging
 from hashlib import sha1
 from saml2.metadata import ENDPOINTS
@@ -857,7 +858,10 @@ def artifact2destination(self, artifact, descriptor):
 
         assert _art[:2] == ARTIFACT_TYPECODE
 
-        endpoint_index = str(int(_art[2:4]))
+        try:
+            endpoint_index = str(int(_art[2:4]))
+        except ValueError:
+            endpoint_index = str(int(hexlify(_art[2:4])))
         entity = self.sourceid[_art[4:24]]
 
         destination = None