feat(clip): 添加 pos 计算

Signed-off-by: YdrMaster <ydrml@hotmail.com>

Author: YdrMaster

models/clip/common-cpu/src/test_infer.rs

@@ -1,7 +1,10 @@
 use crate::{Operators, Weights};
-use clip::{ClipArgs, ClipMeta, ClipStorage, ClipWorker, Image};
-use gguf::GGufModel;
-use operators::common_cpu::{Cpu, ThisThread};
+use clip::{ClipArgs, ClipMeta, ClipStorage, ClipWorker, Image, Tensor};
+use gguf::{ggml_quants::digit_layout::types as ty, GGufModel};
+use operators::{
+    common_cpu::{Cpu, ThisThread},
+    Blob,
+};
 use std::time::Instant;
 use test_utils::Inference;
 
@@ -50,21 +53,51 @@ fn test_infer() {
         .launch(
             ClipArgs {
                 raw: whole.to_nchw(),
+                pos: pos70(whole.shape(), d_patch).map_slice(),
             },
             &mut [],
             &ThisThread,
         )
         .unwrap();
 
     if let Some(patches) = slices.patches_nchw() {
+        let &[_, 3, h, w] = patches.shape() else {
+            unreachable!()
+        };
         worker
             .launch(
                 ClipArgs {
                     raw: patches.map_slice(),
+                    pos: pos70([w, h], d_patch).map_slice(),
                 },
                 &mut [],
                 &ThisThread,
             )
             .unwrap();
     }
 }
+
+fn pos70([w, h]: [usize; 2], d_patch: usize) -> Tensor<Blob> {
+    let pos_w = w / d_patch;
+    let pos_h = h / d_patch;
+    let mut bucket_corrds_h = [0; 70];
+    let mut bucket_corrds_w = [0; 70];
+    for i in 0..pos_w {
+        bucket_corrds_w[i] = ((70 * i) as f64 / pos_w as f64) as _;
+    }
+    for i in 0..pos_h {
+        bucket_corrds_h[i] = ((70 * i) as f64 / pos_h as f64) as _;
+    }
+
+    let mut ans = Tensor::new(ty::U32, &[pos_w * pos_h]).map(Blob::new);
+    let (&mut [], data, &mut []) = (unsafe { ans.get_mut().align_to_mut::<u32>() }) else {
+        panic!()
+    };
+
+    let f = |i, d| ((70 * i) as f64 / d as f64) as u32;
+    for i in 0..pos_h * pos_w {
+        data[i] = f(i / pos_w, pos_h) * 70 + f(i % pos_w, pos_w);
+    }
+
+    ans
+}

models/clip/common/src/args.rs

@@ -4,4 +4,6 @@ use tensor::Tensor;
 pub struct Args<'a, H: Hardware> {
     /// shape: [n, c, h, w]
     pub raw: Tensor<&'a [H::Byte]>,
+    /// shape: [h x w]
+    pub pos: Tensor<&'a [H::Byte]>,
 }

models/clip/common/src/compute.rs

@@ -64,7 +64,7 @@ where
         QA: QueueAlloc<Hardware = Ops::Hardware>,
     {
         let time = Instant::now();
-        let Args { raw } = args;
+        let Args { raw, .. } = args;
         let queue = queue_alloc.queue();
 
         let ClipMeta { dt_embd, .. } = self.meta;
@@ -80,6 +80,8 @@ where
         let mut embd = Tensor::new(dt_embd, &[n, m, h / hk, w / wk]).map(|s| queue_alloc.alloc(s));
         self.conv(&mut embd, &raw, &k, &b, workspace, queue_alloc)?;
 
+        let _embd = embd.merge(2..4).unwrap().transpose(&[2, 1]);
+
         if self.debug {
             println!("encode {n} x {h} x {w} image in {:?}", time.elapsed());
         }