Merge branch 'develop'

DaniFdezAlvarez · DaniFdezAlvarez · commit 53a4fcc018f9 · 2025-09-17T16:30:14.000+02:00
diff --git a/setup.py b/setup.py
@@ -8,12 +8,12 @@ def read(file_path):
 setup(
   name = 'shexer',
   packages = find_packages(exclude=["*.local_code.*"]), # this must be the same as the name above
-  version = '2.6.5',
+  version = '2.6.5.1',
   description = 'Automatic schema extraction for RDF graphs',
   author = 'Daniel Fernandez-Alvarez',
   author_email = 'danifdezalvarez@gmail.com',
   url = 'https://github.com/DaniFdezAlvarez/shexer',
-  download_url = 'https://github.com/DaniFdezAlvarez/shexer/archive/2.6.5.tar.gz',
+  download_url = 'https://github.com/DaniFdezAlvarez/shexer/archive/2.6.5.1.tar.gz',
   keywords = ['testing', 'shexer', 'shexerp3', "rdf", "shex", "shacl", "schema"],
   long_description = read('README.md'),
   long_description_content_type='text/markdown',
diff --git a/shexer/consts.py b/shexer/consts.py
@@ -48,4 +48,7 @@
             "http://www.w3.org/2000/01/rdf-schema#": "rdfs",
             "http://www.w3.org/2001/XMLSchema#": "xsd",
             "http://xmlns.com/foaf/0.1/": "foaf"
-            }
+            }
+
+# WESO-SHAPES-ONTO
+FREQ_PROP = "http://weso.es/shexer/ontology/ratio_property_usage"
diff --git a/shexer/core/shexing/class_shexer.py b/shexer/core/shexing/class_shexer.py
@@ -1,6 +1,6 @@
 import json
 
-from shexer.consts import RDF_TYPE, SHAPES_DEFAULT_NAMESPACE
+from shexer.consts import RDF_TYPE, SHAPES_DEFAULT_NAMESPACE, FREQ_PROP
 from shexer.core.shexing.strategy.direct_shexing_strategy import DirectShexingStrategy
 from shexer.core.shexing.strategy.direct_and_inverse_shexing_strategy import DirectAndInverseShexingStrategy
 from shexer.utils.target_elements import determine_original_target_nodes_if_needed
@@ -18,7 +18,8 @@ def __init__(self, class_counts_dict, class_profile_dict=None, class_profile_jso
                  allow_opt_cardinality=True, disable_exact_cardinality=False,
                  shapes_namespace=SHAPES_DEFAULT_NAMESPACE, inverse_paths=False,
                  decimals=-1, instances_report_mode=RATIO_INSTANCES, detect_minimal_iri=False,
-                 class_min_iris_dict=None, allow_redundant_or=False, shape_names_dict=None):
+                 class_min_iris_dict=None, allow_redundant_or=False, shape_names_dict=None, frequency_property=FREQ_PROP,
+                 comments_to_annotations=False):
         self._class_counts_dict = class_counts_dict
         self._class_profile_dict = class_profile_dict if class_profile_dict is not None else self._load_class_profile_dict_from_file(
             class_profile_json_file)
@@ -42,6 +43,8 @@ def __init__(self, class_counts_dict, class_profile_dict=None, class_profile_jso
         self._detect_minimal_iri = detect_minimal_iri
         self._allow_redundant_or = allow_redundant_or
         self._shape_names_dict = shape_names_dict if shape_names_dict is not None else {}
+        self._frequency_property = frequency_property
+        self._comments_to_annotations = comments_to_annotations
 
         self._original_target_nodes = determine_original_target_nodes_if_needed(remove_empty_shapes=remove_empty_shapes,
                                                                                 original_target_classes=original_target_classes,
diff --git a/shexer/core/shexing/class_shexer_fed_sources.py b/shexer/core/shexing/class_shexer_fed_sources.py
@@ -1,6 +1,5 @@
 from shexer.core.shexing.class_shexer import ClassShexer
-from shexer.consts import RDF_TYPE, SHAPES_DEFAULT_NAMESPACE
-from shexer.consts import RATIO_INSTANCES
+from shexer.consts import RDF_TYPE, SHAPES_DEFAULT_NAMESPACE, FREQ_PROP, RATIO_INSTANCES
 from shexer.core.instances.pconsts import FEDERATION_TAG_MARK
 
 _COMMENT_FED_PROPERTY = "# Constraint only observed in {}"
@@ -15,13 +14,15 @@ def __init__(self, class_counts_dict, class_profile_dict=None, class_profile_jso
                  disable_comments=False, namespaces_dict=None, tolerance_to_keep_similar_rules=0,
                  allow_opt_cardinality=True, disable_exact_cardinality=False, shapes_namespace=SHAPES_DEFAULT_NAMESPACE,
                  inverse_paths=False, decimals=-1, instances_report_mode=RATIO_INSTANCES, detect_minimal_iri=False,
-                 class_min_iris_dict=None, allow_redundant_or=False, fed_sources=None, shape_names_dict=None):
+                 class_min_iris_dict=None, allow_redundant_or=False, fed_sources=None, shape_names_dict=None,
+                 frequency_property=FREQ_PROP, comments_to_annotations=False):
         super().__init__(class_counts_dict, class_profile_dict, class_profile_json_file, remove_empty_shapes,
                          original_target_classes, original_shape_map, discard_useless_constraints_with_positive_closure,
                          keep_less_specific, all_compliant_mode, instantiation_property, disable_or_statements,
                          disable_comments, namespaces_dict, tolerance_to_keep_similar_rules, allow_opt_cardinality,
                          disable_exact_cardinality, shapes_namespace, inverse_paths, decimals, instances_report_mode,
-                         detect_minimal_iri, class_min_iris_dict, allow_redundant_or, shape_names_dict)
+                         detect_minimal_iri, class_min_iris_dict, allow_redundant_or, shape_names_dict,
+                         frequency_property, comments_to_annotations)
         self._fed_sources = fed_sources
 
     def shex_classes(self, acceptance_threshold=0,
diff --git a/shexer/core/shexing/strategy/abstract_shexing_strategy.py b/shexer/core/shexing/strategy/abstract_shexing_strategy.py
@@ -36,7 +36,10 @@ def __init__(self, class_shexer):
         self._statement_serializer_factory = StSerializerFactory(freq_mode=class_shexer._instances_report_mode,
                                                                  decimals=class_shexer._decimals,
                                                                  instantiation_property_str=self._instantiation_property_str,
-                                                                 disable_comments=self._disable_comments)
+                                                                 disable_comments=self._disable_comments,
+                                                                 frequency_property=self._class_shexer._frequency_property,
+                                                                 namespaces_dict=self._namespaces_dict,
+                                                                 comments_to_annotations=class_shexer._comments_to_annotations)
 
 
     def yield_base_shapes(self, acceptance_threshold):
@@ -99,7 +102,7 @@ def _change_statement_cardinality_to_all_compliant(self, statement):
         statement.cardinality = OPT_CARDINALITY if \
             self._allow_opt_cardinality and statement.cardinality == 1 \
             else KLEENE_CLOSURE
-        statement.probability = 1
+        # statement.probability = 1
 
 
     @staticmethod
diff --git a/shexer/io/shex/formater/statement_serializers/base_statement_serializer.py b/shexer/io/shex/formater/statement_serializers/base_statement_serializer.py
@@ -3,29 +3,46 @@
 from shexer.model.const_elem_types import IRI_ELEM_TYPE, BNODE_ELEM_TYPE, NONLITERAL_ELEM_TYPE
 from shexer.model.shape import STARTING_CHAR_FOR_SHAPE_NAME
 from shexer.utils.shapes import prefixize_shape_name_if_possible
+from shexer.utils.uri import prefixize_uri_if_possible
+from shexer.consts import FREQ_PROP
 
 _INVERSE_SENSE_SHEXC = "^"
+_ANNOTATION_BEGIN = "//"
+_FREQUENCY_PATTERN = "{:.3f}"
+
 
 class BaseStatementSerializer(object):
 
-    def __init__(self, instantiation_property_str, frequency_serializer, disable_comments=False, is_inverse=False):
+    def __init__(self, instantiation_property_str, frequency_serializer, disable_comments=False, is_inverse=False,
+                 frequency_property=FREQ_PROP,
+                 namespaces_dict=None,
+                 comments_to_annotations=False):
         self._instantiation_property_str = instantiation_property_str
         self._disable_comments = disable_comments
         self._is_inverse = is_inverse
         self._frequency_serializer = frequency_serializer
+        self._frequency_property = frequency_property
+        self._namespaces_dict=namespaces_dict
+        self._comments_to_annotations = comments_to_annotations
 
-    def serialize_statement_with_indent_level(self, a_statement, is_last_statement_of_shape, namespaces_dict):
+    def serialize_statement_with_indent_level(self, a_statement, is_last_statement_of_shape):
         tuples_line_indent = []
-        st_property = BaseStatementSerializer.tune_token(a_statement.st_property, namespaces_dict)
+        st_property = BaseStatementSerializer.tune_token(a_statement.st_property, self._namespaces_dict)
         st_target_element = self.str_of_target_element(target_element=a_statement.st_type,
-                                                       st_property=a_statement.st_property,
-                                                       namespaces_dict=namespaces_dict)
+                                                       st_property=a_statement.st_property)
         cardinality = BaseStatementSerializer.cardinality_representation(
             statement=a_statement,
             out_of_comment=True)
-        result = self._sense_flag() + st_property + SPACES_GAP_BETWEEN_TOKENS + st_target_element + SPACES_GAP_BETWEEN_TOKENS + \
-                 cardinality + BaseStatementSerializer.closure_of_statement(is_last_statement_of_shape)
-
+        if self._comments_to_annotations:
+            annotations = self._build_constraint_annotations(a_statement)
+            result = self._sense_flag() + st_property + SPACES_GAP_BETWEEN_TOKENS + st_target_element + SPACES_GAP_BETWEEN_TOKENS + \
+                     cardinality + \
+                     SPACES_GAP_BETWEEN_TOKENS + annotations + SPACES_GAP_BETWEEN_TOKENS + \
+                     BaseStatementSerializer.closure_of_statement(is_last_statement_of_shape)
+        else:
+            result = self._sense_flag() + st_property + SPACES_GAP_BETWEEN_TOKENS + st_target_element + SPACES_GAP_BETWEEN_TOKENS + \
+                     cardinality + \
+                     BaseStatementSerializer.closure_of_statement(is_last_statement_of_shape)
         if a_statement.cardinality not in [KLEENE_CLOSURE, OPT_CARDINALITY] and not self._disable_comments:
             result += BaseStatementSerializer.adequate_amount_of_final_spaces(result)
             result += a_statement.probability_representation()
@@ -36,17 +53,26 @@ def serialize_statement_with_indent_level(self, a_statement, is_last_statement_o
 
         return tuples_line_indent
 
-    def str_of_target_element(self, target_element, st_property, namespaces_dict):
+    def _build_constraint_annotations(self, a_statement):
+        return SPACES_GAP_BETWEEN_TOKENS.join((_ANNOTATION_BEGIN,
+                                              prefixize_uri_if_possible(target_uri=self._frequency_property,
+                                                                        namespaces_prefix_dict=self._namespaces_dict,
+                                                                        corners=False),
+                                              self._format_frequency(a_statement.probability)
+                                              ))
+
+    def _format_frequency(self, frequency_raw_number):
+        return _FREQUENCY_PATTERN.format(frequency_raw_number)
+    def str_of_target_element(self, target_element, st_property):
         """
         Special treatment for instantiation_property. We build a value set with an specific URI
         :param target_element:
         :param st_property:
-        :param namespaces_dict:
         :return:
         """
         if st_property == self._instantiation_property_str:
-            return "[" + BaseStatementSerializer.tune_token(target_element, namespaces_dict) + "]"
-        return BaseStatementSerializer.tune_token(target_element, namespaces_dict)
+            return "[" + BaseStatementSerializer.tune_token(target_element, self._namespaces_dict) + "]"
+        return BaseStatementSerializer.tune_token(target_element, self._namespaces_dict)
 
     @staticmethod
     def tune_token(a_token, namespaces_dict):
diff --git a/shexer/io/shex/formater/statement_serializers/fixed_prop_choice_statement_serializer.py b/shexer/io/shex/formater/statement_serializers/fixed_prop_choice_statement_serializer.py
@@ -1,23 +1,27 @@
 from shexer.io.shex.formater.statement_serializers.base_statement_serializer import BaseStatementSerializer
 from shexer.io.shex.formater.consts import SPACES_GAP_BETWEEN_TOKENS, KLEENE_CLOSURE, OPT_CARDINALITY
+from shexer.consts import FREQ_PROP
 
 
 class FixedPropChoiceStatementSerializer(BaseStatementSerializer):
 
-    def __init__(self, instantiation_property_str, frequency_serializer, disable_comments=False, is_inverse=False):
+    def __init__(self, instantiation_property_str, frequency_serializer, disable_comments=False, is_inverse=False,
+                 frequency_property=FREQ_PROP, namespaces_dict=None, comments_to_annotations=False):
         super(FixedPropChoiceStatementSerializer, self).__init__(instantiation_property_str=instantiation_property_str,
                                                                  disable_comments=disable_comments,
                                                                  is_inverse=is_inverse,
-                                                                 frequency_serializer=frequency_serializer)
+                                                                 frequency_serializer=frequency_serializer,
+                                                                 frequency_property=frequency_property,
+                                                                 namespaces_dict=namespaces_dict,
+                                                                 comments_to_annotations=comments_to_annotations)
 
-    def serialize_statement_with_indent_level(self, a_statement, is_last_statement_of_shape, namespaces_dict):
+    def serialize_statement_with_indent_level(self, a_statement, is_last_statement_of_shape):
         tuples_line_indent = []
-        st_property = BaseStatementSerializer.tune_token(a_statement.st_property, namespaces_dict)
+        st_property = BaseStatementSerializer.tune_token(a_statement.st_property, self._namespaces_dict)
         st_target_elements = []
         for a_type in a_statement.st_types:
             st_target_elements.append(self.str_of_target_element(target_element=a_type,
-                                                                 st_property=a_statement.st_property,
-                                                                 namespaces_dict=namespaces_dict))
+                                                                 st_property=a_statement.st_property))
 
         content_line = st_property + SPACES_GAP_BETWEEN_TOKENS
         content_line += (SPACES_GAP_BETWEEN_TOKENS + "OR" + SPACES_GAP_BETWEEN_TOKENS).join(st_target_elements)
diff --git a/shexer/io/shex/formater/statement_serializers/st_serializers_factory.py b/shexer/io/shex/formater/statement_serializers/st_serializers_factory.py
@@ -14,30 +14,42 @@ class StSerializerFactory(object):
 
     """
 
-    def __init__(self, freq_mode, decimals, instantiation_property_str, disable_comments):
+    def __init__(self, freq_mode, decimals, instantiation_property_str, disable_comments, frequency_property, namespaces_dict, comments_to_annotations):
         self._freq_serializer = self._build_freq_serializer(freq_mode=freq_mode,
                                                             decimals=decimals)
 
         self._direct_base = BaseStatementSerializer(
                 instantiation_property_str=instantiation_property_str,
                 disable_comments=disable_comments,
                 is_inverse=False,
-                frequency_serializer=self._freq_serializer)
+                frequency_serializer=self._freq_serializer,
+                frequency_property=frequency_property,
+                namespaces_dict=namespaces_dict,
+                comments_to_annotations=comments_to_annotations)
         self._inverse_base = BaseStatementSerializer(
                 instantiation_property_str=instantiation_property_str,
                 disable_comments=disable_comments,
                 is_inverse=True,
-                frequency_serializer=self._freq_serializer)
+                frequency_serializer=self._freq_serializer,
+                frequency_property=frequency_property,
+                namespaces_dict=namespaces_dict,
+                comments_to_annotations=comments_to_annotations)
         self._direct_choice = FixedPropChoiceStatementSerializer(
                 instantiation_property_str=instantiation_property_str,
                 disable_comments=disable_comments,
                 is_inverse=False,
-                frequency_serializer=self._freq_serializer)
+                frequency_serializer=self._freq_serializer,
+                frequency_property=frequency_property,
+                namespaces_dict=namespaces_dict,
+                comments_to_annotations=comments_to_annotations)
         self._inverse_choice = FixedPropChoiceStatementSerializer(
                 instantiation_property_str=instantiation_property_str,
                 disable_comments=disable_comments,
                 is_inverse=True,
-                frequency_serializer=self._freq_serializer)
+                frequency_serializer=self._freq_serializer,
+                frequency_property=frequency_property,
+                namespaces_dict=namespaces_dict,
+                comments_to_annotations=comments_to_annotations)
 
     def get_base_serializer(self, is_inverse):
         return self._direct_base if not is_inverse else self._inverse_base
diff --git a/shexer/model/statement.py b/shexer/model/statement.py
@@ -18,8 +18,7 @@ def __init__(self, st_property, st_type, cardinality, n_occurences,
     def get_tuples_to_serialize_line_indent_level(self, is_last_statement_of_shape, namespaces_dict):
         return self._serializer_object.\
             serialize_statement_with_indent_level(a_statement=self,
-                                                  is_last_statement_of_shape= is_last_statement_of_shape,
-                                                  namespaces_dict=namespaces_dict)
+                                                  is_last_statement_of_shape= is_last_statement_of_shape)
 
     def probability_representation(self):
         return self._serializer_object.probability_representation(self)
diff --git a/shexer/shaper.py b/shexer/shaper.py
@@ -2,7 +2,7 @@
 
 from shexer.consts import SHEXC, SHACL_TURTLE, NT, TSV_SPO, N3, TURTLE, TURTLE_ITER, \
     RDF_XML, FIXED_SHAPE_MAP, JSON_LD, RDF_TYPE, SHAPES_DEFAULT_NAMESPACE, ZIP, GZ, XZ, \
-    ALL_EXAMPLES, CONSTRAINT_EXAMPLES, SHAPE_EXAMPLES
+    ALL_EXAMPLES, CONSTRAINT_EXAMPLES, SHAPE_EXAMPLES, FREQ_PROP
 from shexer.utils.factories.class_profiler_factory import get_class_profiler
 from shexer.utils.factories.instance_tracker_factory import get_instance_tracker
 from shexer.utils.factories.class_shexer_factory import get_class_shexer
@@ -64,7 +64,9 @@ def __init__(self, target_classes=None,
                  allow_redundant_or=False,
                  instances_cap=-1,
                  examples_mode=None,
-                 federated_sources=None  # could be a list
+                 federated_sources=None,  # could be a list
+                 comments_to_annotations=False,
+                 frequency_property=FREQ_PROP
                  ):
         """
 
@@ -197,6 +199,9 @@ def __init__(self, target_classes=None,
         self._namespaces_for_qualifier_props = namespaces_for_qualifier_props
         self._shapes_namespace = shapes_namespace
 
+        self._comments_to_annotations=comments_to_annotations
+        self._frequency_property=frequency_property
+
         #The following two atts are used for optimizations
         self._built_remote_graph = get_remote_graph_if_needed(endpoint_url=url_endpoint,
                                                               store_locally=not disable_endpoint_cache)
@@ -356,7 +361,9 @@ def _build_class_shexer(self):
                                 class_min_iris=self._class_min_iris,
                                 allow_redundant_or=self._allow_redundant_or,
                                 federated_sources=self._federated_sources,
-                                shape_names=self._shape_names)
+                                shape_names=self._shape_names,
+                                frequency_property=self._frequency_property,
+                                comments_to_annotations=self._comments_to_annotations)
 
     def _build_shapes_serializer(self, target_file, string_return, output_format, rdfconfig_directory, verbose):
         return get_shape_serializer(shapes_list=self._shape_list,
diff --git a/shexer/utils/factories/class_shexer_factory.py b/shexer/utils/factories/class_shexer_factory.py
diff --git a/test/t_files/annotations/g1_freq_annotations.shex b/test/t_files/annotations/g1_freq_annotations.shex
diff --git a/test/test_annotations.py b/test/test_annotations.py
