test: add 2-3 additional tests (#460)

* 2-3 additional tests.

* Removing non-ASCII content in the main source files + marking test files
as UTF-8.

* Reformat.

* Reformat

---------

Co-authored-by: Daniel Lemire <[email protected]>

Author: lemire

include/ada/ada_idna.h

@@ -110,7 +110,6 @@ namespace ada::idna {
 // this function. We also do not trim control characters. We also assume that
 // the input is not empty. We return "" on error.
 //
-// Example: "www.öbb.at" -> "www.xn--bb-eka.at"
 //
 // This function may accept or even produce invalid domains.
 std::string to_ascii(std::string_view ut8_string);

include/ada/unicode.h

@@ -24,10 +24,10 @@ namespace ada::unicode {
  * Given a domain, we need to identify its labels.
  * They are separated by label-separators:
  *
- * U+002E ( . ) FULL STOP
- * U+FF0E ( ． ) FULLWIDTH FULL STOP
- * U+3002 ( 。 ) IDEOGRAPHIC FULL STOP
- * U+FF61 ( ｡ ) HALFWIDTH IDEOGRAPHIC FULL STOP
+ * U+002E (.) FULL STOP
+ * U+FF0E FULLWIDTH FULL STOP
+ * U+3002 IDEOGRAPHIC FULL STOP
+ * U+FF61 HALFWIDTH IDEOGRAPHIC FULL STOP
  *
  * They are all mapped to U+002E.
  *

include/ada/url-inl.h

@@ -80,8 +80,8 @@ size_t url::get_pathname_length() const noexcept { return path.size(); }
     out.host_end = out.host_start;
 
     if (!has_opaque_path && checkers::begins_with(path, "//")) {
-      // If url’s host is null, url does not have an opaque path, url’s path’s
-      // size is greater than 1, and url’s path[0] is the empty string, then
+      // If url's host is null, url does not have an opaque path, url's path's
+      // size is greater than 1, and url's path[0] is the empty string, then
       // append U+002F (/) followed by U+002E (.) to output.
       running_index = out.protocol_end + 2;
     } else {
@@ -195,8 +195,8 @@ inline void url::copy_scheme(const ada::url &u) {
       output += ":" + get_port();
     }
   } else if (!has_opaque_path && checkers::begins_with(path, "//")) {
-    // If url’s host is null, url does not have an opaque path, url’s path’s
-    // size is greater than 1, and url’s path[0] is the empty string, then
+    // If url's host is null, url does not have an opaque path, url's path's
+    // size is greater than 1, and url's path[0] is the empty string, then
     // append U+002F (/) followed by U+002E (.) to output.
     output += "/.";
   }

include/ada/url.h

@@ -45,48 +45,48 @@ struct url : url_base {
 
   /**
    * @private
-   * A URL’s username is an ASCII string identifying a username. It is initially
+   * A URL's username is an ASCII string identifying a username. It is initially
    * the empty string.
    */
   std::string username{};
 
   /**
    * @private
-   * A URL’s password is an ASCII string identifying a password. It is initially
+   * A URL's password is an ASCII string identifying a password. It is initially
    * the empty string.
    */
   std::string password{};
 
   /**
    * @private
-   * A URL’s host is null or a host. It is initially null.
+   * A URL's host is null or a host. It is initially null.
    */
   std::optional<std::string> host{};
 
   /**
    * @private
-   * A URL’s port is either null or a 16-bit unsigned integer that identifies a
+   * A URL's port is either null or a 16-bit unsigned integer that identifies a
    * networking port. It is initially null.
    */
   std::optional<uint16_t> port{};
 
   /**
    * @private
-   * A URL’s path is either an ASCII string or a list of zero or more ASCII
+   * A URL's path is either an ASCII string or a list of zero or more ASCII
    * strings, usually identifying a location.
    */
   std::string path{};
 
   /**
    * @private
-   * A URL’s query is either null or an ASCII string. It is initially null.
+   * A URL's query is either null or an ASCII string. It is initially null.
    */
   std::optional<std::string> query{};
 
   /**
    * @private
-   * A URL’s fragment is either null or an ASCII string that can be used for
-   * further processing on the resource the URL’s other components identify. It
+   * A URL's fragment is either null or an ASCII string that can be used for
+   * further processing on the resource the URL's other components identify. It
    * is initially null.
    */
   std::optional<std::string> hash{};
@@ -111,23 +111,23 @@ struct url : url_base {
   [[nodiscard]] ada_really_inline std::string get_href() const noexcept;
 
   /**
-   * The origin getter steps are to return the serialization of this’s URL’s
+   * The origin getter steps are to return the serialization of this's URL's
    * origin. [HTML]
    * @return a newly allocated string.
    * @see https://url.spec.whatwg.org/#concept-url-origin
    */
   [[nodiscard]] std::string get_origin() const noexcept override;
 
   /**
-   * The protocol getter steps are to return this’s URL’s scheme, followed by
+   * The protocol getter steps are to return this's URL's scheme, followed by
    * U+003A (:).
    * @return a newly allocated string.
    * @see https://url.spec.whatwg.org/#dom-url-protocol
    */
   [[nodiscard]] std::string get_protocol() const noexcept;
 
   /**
-   * Return url’s host, serialized, followed by U+003A (:) and url’s port,
+   * Return url's host, serialized, followed by U+003A (:) and url's port,
    * serialized.
    * When there is no host, this function returns the empty string.
    * @return a newly allocated string.
@@ -136,7 +136,7 @@ struct url : url_base {
   [[nodiscard]] std::string get_host() const noexcept;
 
   /**
-   * Return this’s URL’s host, serialized.
+   * Return this's URL's host, serialized.
    * When there is no host, this function returns the empty string.
    * @return a newly allocated string.
    * @see https://url.spec.whatwg.org/#dom-url-hostname
@@ -145,7 +145,7 @@ struct url : url_base {
 
   /**
    * The pathname getter steps are to return the result of URL path serializing
-   * this’s URL.
+   * this's URL.
    * @return a newly allocated string.
    * @see https://url.spec.whatwg.org/#dom-url-pathname
    */
@@ -160,14 +160,14 @@ struct url : url_base {
   ada_really_inline size_t get_pathname_length() const noexcept;
 
   /**
-   * Return U+003F (?), followed by this’s URL’s query.
+   * Return U+003F (?), followed by this's URL's query.
    * @return a newly allocated string.
    * @see https://url.spec.whatwg.org/#dom-url-search
    */
   [[nodiscard]] std::string get_search() const noexcept;
 
   /**
-   * The username getter steps are to return this’s URL’s username.
+   * The username getter steps are to return this's URL's username.
    * @return a constant reference to the underlying string.
    * @see https://url.spec.whatwg.org/#dom-url-username
    */
@@ -233,21 +233,21 @@ struct url : url_base {
   bool set_href(const std::string_view input);
 
   /**
-   * The password getter steps are to return this’s URL’s password.
+   * The password getter steps are to return this's URL's password.
    * @return a constant reference to the underlying string.
    * @see https://url.spec.whatwg.org/#dom-url-password
    */
   [[nodiscard]] const std::string &get_password() const noexcept;
 
   /**
-   * Return this’s URL’s port, serialized.
+   * Return this's URL's port, serialized.
    * @return a newly constructed string representing the port.
    * @see https://url.spec.whatwg.org/#dom-url-port
    */
   [[nodiscard]] std::string get_port() const noexcept;
 
   /**
-   * Return U+0023 (#), followed by this’s URL’s fragment.
+   * Return U+0023 (#), followed by this's URL's fragment.
    * @return a newly constructed string representing the hash.
    * @see https://url.spec.whatwg.org/#dom-url-hash
    */
@@ -333,7 +333,7 @@ struct url : url_base {
   [[nodiscard]] bool parse_opaque_host(std::string_view input);
 
   /**
-   * A URL’s scheme is an ASCII string that identifies the type of URL and can
+   * A URL's scheme is an ASCII string that identifies the type of URL and can
    * be used to dispatch a URL for further processing after parsing. It is
    * initially the empty string. We only set non_special_scheme when the scheme
    * is non-special, otherwise we avoid constructing string.

include/ada/url_aggregator-inl.h

@@ -276,7 +276,7 @@ inline void url_aggregator::update_base_pathname(const std::string_view input) {
 
   if (begins_with_dashdash && !has_opaque_path && !has_authority() &&
       !has_dash_dot()) {
-    // If url’s host is null, url does not have an opaque path, url’s path’s
+    // If url's host is null, url does not have an opaque path, url's path's
     // size is greater than 1, then append U+002F (/) followed by U+002E (.) to
     // output.
     buffer.insert(components.pathname_start, "/.");
@@ -809,8 +809,8 @@ inline bool url_aggregator::has_port() const noexcept {
 }
 
 inline bool url_aggregator::has_dash_dot() const noexcept {
-  // If url’s host is null, url does not have an opaque path, url’s path’s size
-  // is greater than 1, and url’s path[0] is the empty string, then append
+  // If url's host is null, url does not have an opaque path, url's path's size
+  // is greater than 1, and url's path[0] is the empty string, then append
   // U+002F (/) followed by U+002E (.) to output.
   ada_log("url_aggregator::has_dash_dot");
   // Performance: instead of doing this potentially expensive check, we could

include/ada/url_aggregator.h

@@ -43,7 +43,7 @@ struct url_aggregator : url_base {
 
   [[nodiscard]] bool has_valid_domain() const noexcept override;
   /**
-   * The origin getter steps are to return the serialization of this’s URL’s
+   * The origin getter steps are to return the serialization of this's URL's
    * origin. [HTML]
    * @return a newly allocated string.
    * @see https://url.spec.whatwg.org/#concept-url-origin
@@ -59,35 +59,35 @@ struct url_aggregator : url_base {
    */
   inline std::string_view get_href() const noexcept;
   /**
-   * The username getter steps are to return this’s URL’s username.
+   * The username getter steps are to return this's URL's username.
    * This function does not allocate memory.
    * @return a lightweight std::string_view.
    * @see https://url.spec.whatwg.org/#dom-url-username
    */
   [[nodiscard]] std::string_view get_username() const noexcept;
   /**
-   * The password getter steps are to return this’s URL’s password.
+   * The password getter steps are to return this's URL's password.
    * This function does not allocate memory.
    * @return a lightweight std::string_view.
    * @see https://url.spec.whatwg.org/#dom-url-password
    */
   [[nodiscard]] std::string_view get_password() const noexcept;
   /**
-   * Return this’s URL’s port, serialized.
+   * Return this's URL's port, serialized.
    * This function does not allocate memory.
    * @return a lightweight std::string_view.
    * @see https://url.spec.whatwg.org/#dom-url-port
    */
   [[nodiscard]] std::string_view get_port() const noexcept;
   /**
-   * Return U+0023 (#), followed by this’s URL’s fragment.
+   * Return U+0023 (#), followed by this's URL's fragment.
    * This function does not allocate memory.
    * @return a lightweight std::string_view..
    * @see https://url.spec.whatwg.org/#dom-url-hash
    */
   [[nodiscard]] std::string_view get_hash() const noexcept;
   /**
-   * Return url’s host, serialized, followed by U+003A (:) and url’s port,
+   * Return url's host, serialized, followed by U+003A (:) and url's port,
    * serialized.
    * This function does not allocate memory.
    * When there is no host, this function returns the empty view.
@@ -96,7 +96,7 @@ struct url_aggregator : url_base {
    */
   [[nodiscard]] std::string_view get_host() const noexcept;
   /**
-   * Return this’s URL’s host, serialized.
+   * Return this's URL's host, serialized.
    * This function does not allocate memory.
    * When there is no host, this function returns the empty view.
    * @return a lightweight std::string_view.
@@ -105,7 +105,7 @@ struct url_aggregator : url_base {
   [[nodiscard]] std::string_view get_hostname() const noexcept;
   /**
    * The pathname getter steps are to return the result of URL path serializing
-   * this’s URL.
+   * this's URL.
    * This function does not allocate memory.
    * @return a lightweight std::string_view.
    * @see https://url.spec.whatwg.org/#dom-url-pathname
@@ -119,14 +119,14 @@ struct url_aggregator : url_base {
    */
   ada_really_inline uint32_t get_pathname_length() const noexcept;
   /**
-   * Return U+003F (?), followed by this’s URL’s query.
+   * Return U+003F (?), followed by this's URL's query.
    * This function does not allocate memory.
    * @return a lightweight std::string_view.
    * @see https://url.spec.whatwg.org/#dom-url-search
    */
   [[nodiscard]] std::string_view get_search() const noexcept;
   /**
-   * The protocol getter steps are to return this’s URL’s scheme, followed by
+   * The protocol getter steps are to return this's URL's scheme, followed by
    * U+003A (:).
    * This function does not allocate memory.
    * @return a lightweight std::string_view.

include/ada/url_base.h

@@ -47,7 +47,7 @@ struct url_base {
   [[nodiscard]] ada_really_inline bool is_special() const noexcept;
 
   /**
-   * The origin getter steps are to return the serialization of this’s URL’s
+   * The origin getter steps are to return the serialization of this's URL's
    * origin. [HTML]
    * @return a newly allocated string.
    * @see https://url.spec.whatwg.org/#concept-url-origin

src/helpers.cpp

@@ -102,8 +102,8 @@ ada_really_inline bool shorten_path(std::string& path,
                                     ada::scheme::type type) noexcept {
   size_t first_delimiter = path.find_first_of('/', 1);
 
-  // Let path be url’s path.
-  // If url’s scheme is "file", path’s size is 1, and path[0] is a normalized
+  // Let path be url's path.
+  // If url's scheme is "file", path's size is 1, and path[0] is a normalized
   // Windows drive letter, then return.
   if (type == ada::scheme::type::FILE &&
       first_delimiter == std::string_view::npos && !path.empty()) {
@@ -113,7 +113,7 @@ ada_really_inline bool shorten_path(std::string& path,
     }
   }
 
-  // Remove path’s last item, if any.
+  // Remove path's last item, if any.
   size_t last_delimiter = path.rfind('/');
   if (last_delimiter != std::string::npos) {
     path.erase(last_delimiter);
@@ -127,8 +127,8 @@ ada_really_inline bool shorten_path(std::string_view& path,
                                     ada::scheme::type type) noexcept {
   size_t first_delimiter = path.find_first_of('/', 1);
 
-  // Let path be url’s path.
-  // If url’s scheme is "file", path’s size is 1, and path[0] is a normalized
+  // Let path be url's path.
+  // If url's scheme is "file", path's size is 1, and path[0] is a normalized
   // Windows drive letter, then return.
   if (type == ada::scheme::type::FILE &&
       first_delimiter == std::string_view::npos && !path.empty()) {
@@ -138,7 +138,7 @@ ada_really_inline bool shorten_path(std::string_view& path,
     }
   }
 
-  // Remove path’s last item, if any.
+  // Remove path's last item, if any.
   if (!path.empty()) {
     size_t slash_loc = path.rfind('/');
     if (slash_loc != std::string_view::npos) {
@@ -551,7 +551,7 @@ ada_really_inline void parse_prepared_path(std::string_view input,
       }
       // Otherwise, if path_buffer is not a single-dot path segment, then:
       else if (!unicode::is_single_dot_path_segment(path_buffer)) {
-        // If url’s scheme is "file", url’s path is empty, and path_buffer is a
+        // If url's scheme is "file", url's path is empty, and path_buffer is a
         // Windows drive letter, then replace the second code point in
         // path_buffer with U+003A (:).
         if (type == ada::scheme::type::FILE && path.empty() &&
@@ -562,7 +562,7 @@ ada_really_inline void parse_prepared_path(std::string_view input,
           path_buffer.remove_prefix(2);
           path.append(path_buffer);
         } else {
-          // Append path_buffer to url’s path.
+          // Append path_buffer to url's path.
           path += '/';
           path.append(path_buffer);
         }