feat: Add support for range mappings proposal (#77)

Author: kdy1

Cargo.toml

@@ -33,6 +33,7 @@ scroll = { version = "0.10.1", features = ["derive"], optional = true }
 data-encoding = "2.3.3"
 debugid = {version = "0.8.0", features = ["serde"] }
 base64-simd = { version = "0.7" }
+bitvec = "1.0.1"
 rustc-hash = "1.1.0"
 
 [build-dependencies]

src/builder.rs

@@ -182,6 +182,7 @@ impl SourceMapBuilder {
     }
 
     /// Adds a new mapping to the builder.
+    #[allow(clippy::too_many_arguments)]
     pub fn add(
         &mut self,
         dst_line: u32,
@@ -190,8 +191,11 @@ impl SourceMapBuilder {
         src_col: u32,
         source: Option<&str>,
         name: Option<&str>,
+        is_range: bool,
     ) -> RawToken {
-        self.add_with_id(dst_line, dst_col, src_line, src_col, source, !0, name)
+        self.add_with_id(
+            dst_line, dst_col, src_line, src_col, source, !0, name, is_range,
+        )
     }
 
     #[allow(clippy::too_many_arguments)]
@@ -204,6 +208,7 @@ impl SourceMapBuilder {
         source: Option<&str>,
         source_id: u32,
         name: Option<&str>,
+        is_range: bool,
     ) -> RawToken {
         let src_id = match source {
             Some(source) => self.add_source_with_id(source, source_id),
@@ -220,12 +225,14 @@ impl SourceMapBuilder {
             src_col,
             src_id,
             name_id,
+            is_range,
         };
         self.tokens.push(raw);
         raw
     }
 
     /// Adds a new mapping to the builder.
+    #[allow(clippy::too_many_arguments)]
     pub fn add_raw(
         &mut self,
         dst_line: u32,
@@ -234,6 +241,7 @@ impl SourceMapBuilder {
         src_col: u32,
         source: Option<u32>,
         name: Option<u32>,
+        is_range: bool,
     ) -> RawToken {
         let src_id = source.unwrap_or(!0);
         let name_id = name.unwrap_or(!0);
@@ -244,6 +252,7 @@ impl SourceMapBuilder {
             src_col,
             src_id,
             name_id,
+            is_range,
         };
         self.tokens.push(raw);
         raw
@@ -261,6 +270,7 @@ impl SourceMapBuilder {
             token.get_source(),
             token.get_src_id(),
             name,
+            token.is_range(),
         )
     }
 

src/decoder.rs

@@ -1,6 +1,9 @@
 use std::io;
 use std::io::{BufReader, Read};
 
+use bitvec::field::BitField;
+use bitvec::order::Lsb0;
+use bitvec::vec::BitVec;
 use serde_json::Value;
 
 use crate::errors::{Error, Result};
@@ -120,6 +123,29 @@ pub fn strip_junk_header(slice: &[u8]) -> io::Result<&[u8]> {
     Ok(&slice[slice.len()..])
 }
 
+/// Decodes range mappping bitfield string into index
+fn decode_rmi(rmi_str: &str, val: &mut BitVec<u8, Lsb0>) -> Result<()> {
+    val.clear();
+    val.resize(rmi_str.len() * 6, false);
+
+    for (idx, &byte) in rmi_str.as_bytes().iter().enumerate() {
+        let byte = match byte {
+            b'A'..=b'Z' => byte - b'A',
+            b'a'..=b'z' => byte - b'a' + 26,
+            b'0'..=b'9' => byte - b'0' + 52,
+            b'+' => 62,
+            b'/' => 63,
+            _ => {
+                fail!(Error::InvalidBase64(byte as char));
+            }
+        };
+
+        val[6 * idx..6 * (idx + 1)].store_le::<u8>(byte);
+    }
+
+    Ok(())
+}
+
 pub fn decode_regular(rsm: RawSourceMap) -> Result<SourceMap> {
     let mut dst_col;
     let mut src_id = 0;
@@ -129,20 +155,28 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result<SourceMap> {
 
     let names = rsm.names.unwrap_or_default();
     let sources = rsm.sources.unwrap_or_default();
+    let range_mappings = rsm.range_mappings.unwrap_or_default();
     let mappings = rsm.mappings.unwrap_or_default();
     let allocation_size = mappings.matches(&[',', ';'][..]).count() + 10;
     let mut tokens = Vec::with_capacity(allocation_size);
 
     let mut nums = Vec::with_capacity(6);
+    let mut rmi = BitVec::new();
 
-    for (dst_line, line) in mappings.split(';').enumerate() {
+    for (dst_line, (line, rmi_str)) in mappings
+        .split(';')
+        .zip(range_mappings.split(';').chain(std::iter::repeat("")))
+        .enumerate()
+    {
         if line.is_empty() {
             continue;
         }
 
         dst_col = 0;
 
-        for segment in line.split(',') {
+        decode_rmi(rmi_str, &mut rmi)?;
+
+        for (line_index, segment) in line.split(',').enumerate() {
             if segment.is_empty() {
                 continue;
             }
@@ -176,13 +210,16 @@ pub fn decode_regular(rsm: RawSourceMap) -> Result<SourceMap> {
                 }
             }
 
+            let is_range = rmi.get(line_index).map(|v| *v).unwrap_or_default();
+
             tokens.push(RawToken {
                 dst_line: dst_line as u32,
                 dst_col,
                 src_line,
                 src_col,
                 src_id: src,
                 name_id: name,
+                is_range,
             });
         }
     }
@@ -319,3 +356,24 @@ fn test_bad_newline() {
         }
     }
 }
+
+#[test]
+fn test_decode_rmi() {
+    fn decode(rmi_str: &str) -> Vec<usize> {
+        let mut out = bitvec::bitvec![u8, Lsb0; 0; 0];
+        decode_rmi(rmi_str, &mut out).expect("failed to decode");
+
+        let mut res = vec![];
+        for (idx, bit) in out.iter().enumerate() {
+            if *bit {
+                res.push(idx);
+            }
+        }
+        res
+    }
+
+    // This is 0-based index of the bits
+    assert_eq!(decode("AAB"), vec![12]);
+    assert_eq!(decode("g"), vec![5]);
+    assert_eq!(decode("Bg"), vec![0, 11]);
+}

src/encoder.rs

@@ -1,5 +1,8 @@
 use std::io::Write;
 
+use bitvec::field::BitField;
+use bitvec::order::Lsb0;
+use bitvec::view::BitView;
 use serde_json::Value;
 
 use crate::errors::Result;
@@ -21,6 +24,78 @@ fn encode_vlq_diff(out: &mut String, a: u32, b: u32) {
     encode_vlq(out, i64::from(a) - i64::from(b))
 }
 
+fn encode_rmi(out: &mut Vec<u8>, data: &mut Vec<u8>) {
+    fn encode_byte(b: u8) -> u8 {
+        match b {
+            0..=25 => b + b'A',
+            26..=51 => b + b'a' - 26,
+            52..=61 => b + b'0' - 52,
+            62 => b'+',
+            63 => b'/',
+            _ => panic!("invalid byte"),
+        }
+    }
+
+    let bits = data.view_bits_mut::<Lsb0>();
+
+    // trim zero at the end
+    let mut last = 0;
+    for (idx, bit) in bits.iter().enumerate() {
+        if *bit {
+            last = idx;
+        }
+    }
+    let bits = &mut bits[..last + 1];
+
+    for byte in bits.chunks(6) {
+        let byte = byte.load::<u8>();
+
+        let encoded = encode_byte(byte);
+
+        out.push(encoded);
+    }
+}
+
+fn serialize_range_mappings(sm: &SourceMap) -> Option<String> {
+    let mut buf = Vec::new();
+    let mut prev_line = 0;
+    let mut had_rmi = false;
+
+    let mut idx_of_first_in_line = 0;
+
+    let mut rmi_data = Vec::<u8>::new();
+
+    for (idx, token) in sm.tokens().enumerate() {
+        if token.is_range() {
+            had_rmi = true;
+
+            let num = idx - idx_of_first_in_line;
+
+            rmi_data.resize(rmi_data.len() + 2, 0);
+
+            let rmi_bits = rmi_data.view_bits_mut::<Lsb0>();
+            rmi_bits.set(num, true);
+        }
+
+        while token.get_dst_line() != prev_line {
+            if had_rmi {
+                encode_rmi(&mut buf, &mut rmi_data);
+                rmi_data.clear();
+            }
+
+            buf.push(b';');
+            prev_line += 1;
+            had_rmi = false;
+            idx_of_first_in_line = idx;
+        }
+    }
+    if had_rmi {
+        encode_rmi(&mut buf, &mut rmi_data);
+    }
+
+    Some(String::from_utf8(buf).expect("invalid utf8"))
+}
+
 fn serialize_mappings(sm: &SourceMap) -> String {
     let mut rv = String::new();
     // dst == minified == generated
@@ -89,6 +164,7 @@ impl Encodable for SourceMap {
             sources_content: if have_contents { Some(contents) } else { None },
             sections: None,
             names: Some(self.names().map(|x| Value::String(x.to_string())).collect()),
+            range_mappings: serialize_range_mappings(self),
             mappings: Some(serialize_mappings(self)),
             x_facebook_offsets: None,
             x_metro_module_paths: None,
@@ -121,6 +197,7 @@ impl Encodable for SourceMapIndex {
                     .collect(),
             ),
             names: None,
+            range_mappings: None,
             mappings: None,
             x_facebook_offsets: None,
             x_metro_module_paths: None,
@@ -139,3 +216,26 @@ impl Encodable for DecodedMap {
         }
     }
 }
+
+#[test]
+fn test_encode_rmi() {
+    fn encode(indices: &[usize]) -> String {
+        let mut out = vec![];
+
+        // Fill with zeros while testing
+        let mut data = vec![0; 256];
+
+        let bits = data.view_bits_mut::<Lsb0>();
+        for &i in indices {
+            bits.set(i, true);
+        }
+
+        encode_rmi(&mut out, &mut data);
+        String::from_utf8(out).unwrap()
+    }
+
+    // This is 0-based index
+    assert_eq!(encode(&[12]), "AAB");
+    assert_eq!(encode(&[5]), "g");
+    assert_eq!(encode(&[0, 11]), "Bg");
+}

src/errors.rs

@@ -45,6 +45,10 @@ pub enum Error {
     InvalidRamBundleEntry,
     /// Tried to operate on a non RAM bundle file
     NotARamBundle,
+    /// Range mapping index is invalid
+    InvalidRangeMappingIndex(data_encoding::DecodeError),
+
+    InvalidBase64(char),
 }
 
 impl From<io::Error> for Error {
@@ -78,6 +82,12 @@ impl From<serde_json::Error> for Error {
     }
 }
 
+impl From<data_encoding::DecodeError> for Error {
+    fn from(err: data_encoding::DecodeError) -> Error {
+        Error::InvalidRangeMappingIndex(err)
+    }
+}
+
 impl error::Error for Error {
     fn cause(&self) -> Option<&dyn error::Error> {
         match *self {
@@ -114,6 +124,8 @@ impl fmt::Display for Error {
             Error::InvalidRamBundleIndex => write!(f, "invalid module index in ram bundle"),
             Error::InvalidRamBundleEntry => write!(f, "invalid ram bundle module entry"),
             Error::NotARamBundle => write!(f, "not a ram bundle"),
+            Error::InvalidRangeMappingIndex(err) => write!(f, "invalid range mapping index: {err}"),
+            Error::InvalidBase64(c) => write!(f, "invalid base64 character: {}", c),
         }
     }
 }

src/jsontypes.rs

@@ -42,6 +42,8 @@ pub struct RawSourceMap {
     pub sections: Option<Vec<RawSection>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub names: Option<Vec<Value>>,
+    #[serde(rename = "rangeMappings", skip_serializing_if = "Option::is_none")]
+    pub range_mappings: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub mappings: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]

src/ram_bundle.rs

@@ -402,6 +402,7 @@ impl<'a> SplitRamBundleModuleIter<'a> {
                 token.get_src_col(),
                 token.get_source(),
                 token.get_name(),
+                false,
             );
             if token.get_source().is_some() && !builder.has_source_contents(raw.src_id) {
                 builder.set_source_contents(