Add patch for tokenizers-0.19+

Author: msimacek

graalpython/lib-graalpython/patches/tokenizers/metadata.toml

@@ -1,3 +1,8 @@
+[[rules]]
+version = '>= 0.19'
+patch = 'tokenizers-0.19-plus.patch'
+license = 'Apache-2.0'
+
 [[rules]]
 version = '== 0.13.3'
 patch = 'tokenizers-0.13.3.patch'

graalpython/lib-graalpython/patches/tokenizers/tokenizers-0.19-plus.patch

@@ -0,0 +1,28 @@
+diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
+index 3f1e713..6dd3c72 100644
+--- a/bindings/python/src/lib.rs
++++ b/bindings/python/src/lib.rs
+@@ -50,14 +50,16 @@ extern "C" fn child_after_fork() {
+ pub fn tokenizers(m: &Bound<'_, PyModule>) -> PyResult<()> {
+     let _ = env_logger::try_init_from_env("TOKENIZERS_LOG");
+ 
++    // GraalPy change: Disable the atfork warning. This triggers a ton of false positives when
++    // jline calls stty and we don't support fork anyway
+     // Register the fork callback
+-    #[cfg(target_family = "unix")]
+-    unsafe {
+-        if !REGISTERED_FORK_CALLBACK {
+-            libc::pthread_atfork(None, None, Some(child_after_fork));
+-            REGISTERED_FORK_CALLBACK = true;
+-        }
+-    }
++    // #[cfg(target_family = "unix")]
++    // unsafe {
++    //     if !REGISTERED_FORK_CALLBACK {
++    //         libc::pthread_atfork(None, None, Some(child_after_fork));
++    //         REGISTERED_FORK_CALLBACK = true;
++    //     }
++    // }
+ 
+     m.add_class::<tokenizer::PyTokenizer>()?;
+     m.add_class::<tokenizer::PyAddedToken>()?;