Add option to commit changes to the database automatically (#53)

Author: Azrenbeth

README.md

@@ -106,6 +106,11 @@ If this flag is set then output the node and edge information for the state_grou
 directed graph built up from the predecessor state_group links. These can be looked
 at in something like Gephi (https://gephi.org)
 
+- -c  
+If this flag is set then the changes the compressor makes will be committed to the
+database. This should be safe to use while synapse is running as it assumes by default
+that the transactions flag is set
+
 ## Using as python library
 
 The compressor can also be built into a python library as it uses PyO3. It can be

compressor_integration_tests/src/lib.rs

@@ -1,8 +1,10 @@
 use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode};
-use postgres::Client;
+use postgres::{fallible_iterator::FallibleIterator, Client};
 use postgres_openssl::MakeTlsConnector;
 use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use state_map::StateMap;
 use std::{borrow::Cow, collections::BTreeMap, fmt};
+use string_cache::DefaultAtom as Atom;
 
 use synapse_compress_state::StateGroupEntry;
 
@@ -105,3 +107,230 @@ impl<'a> fmt::Display for PGEscape<'a> {
         write!(f, "{}{}{}", delim, self.0, delim)
     }
 }
+
+/// Checks whether the state at each state group is the same as what the map thinks it should be
+///
+/// i.e. when synapse tries to work out the state for a given state group by looking at
+/// the database. Will the state it gets be the same as what the map thinks it should be
+pub fn database_collapsed_states_match_map(
+    state_group_map: &BTreeMap<i64, StateGroupEntry>,
+) -> bool {
+    for sg in state_group_map.keys() {
+        let map_state = collapse_state_with_map(state_group_map, *sg);
+        let database_state = collapse_state_with_database(*sg);
+        if map_state != database_state {
+            println!("database state {} doesn't match", sg);
+            println!("expected {:?}", map_state);
+            println!("but found {:?}", database_state);
+            return false;
+        }
+    }
+    true
+}
+
+/// Gets the full state for a given group from the map (of deltas)
+fn collapse_state_with_map(
+    map: &BTreeMap<i64, StateGroupEntry>,
+    state_group: i64,
+) -> StateMap<Atom> {
+    let mut entry = &map[&state_group];
+    let mut state_map = StateMap::new();
+
+    let mut stack = vec![state_group];
+
+    while let Some(prev_state_group) = entry.prev_state_group {
+        stack.push(prev_state_group);
+        if !map.contains_key(&prev_state_group) {
+            panic!("Missing {}", prev_state_group);
+        }
+        entry = &map[&prev_state_group];
+    }
+
+    for sg in stack.iter().rev() {
+        state_map.extend(
+            map[sg]
+                .state_map
+                .iter()
+                .map(|((t, s), e)| ((t, s), e.clone())),
+        );
+    }
+
+    state_map
+}
+
+fn collapse_state_with_database(state_group: i64) -> StateMap<Atom> {
+    // connect to the database
+    let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
+    builder.set_verify(SslVerifyMode::NONE);
+    let connector = MakeTlsConnector::new(builder.build());
+
+    let mut client = Client::connect(DB_URL, connector).unwrap();
+
+    // Gets the delta for a specific state group
+    let query_deltas = r#"
+        SELECT m.id, type, state_key, s.event_id
+        FROM state_groups AS m
+        LEFT JOIN state_groups_state AS s ON (m.id = s.state_group)
+        WHERE m.id = $1
+    "#;
+
+    // If there is no delta for that specific state group, then we still want to find
+    // the predecessor (so have split this into a different query)
+    let query_pred = r#"
+        SELECT prev_state_group
+        FROM state_group_edges 
+        WHERE state_group = $1
+    "#;
+
+    let mut state_map: StateMap<Atom> = StateMap::new();
+
+    let mut next_group = Some(state_group);
+
+    while let Some(sg) = next_group {
+        // get predecessor from state_group_edges
+        let mut pred = client.query_raw(query_pred, &[sg]).unwrap();
+
+        // set next_group to predecessor
+        next_group = match pred.next().unwrap() {
+            Some(p) => p.get(0),
+            None => None,
+        };
+
+        // if there was a predecessor then assert that it is unique
+        if next_group.is_some() {
+            assert!(pred.next().unwrap().is_none());
+        }
+        drop(pred);
+
+        let mut rows = client.query_raw(query_deltas, &[sg]).unwrap();
+
+        while let Some(row) = rows.next().unwrap() {
+            // Copy the single delta from the predecessor stored in this row
+            if let Some(etype) = row.get::<_, Option<String>>(1) {
+                let key = &row.get::<_, String>(2);
+
+                // only insert if not overriding existing entry
+                // this is because the newer delta is found FIRST
+                if state_map.get(&etype, key).is_none() {
+                    state_map.insert(&etype, key, row.get::<_, String>(3).into());
+                }
+            }
+        }
+    }
+
+    state_map
+}
+
+/// Check whether predecessors and deltas stored in the database are the same as in the map
+pub fn database_structure_matches_map(state_group_map: &BTreeMap<i64, StateGroupEntry>) -> bool {
+    // connect to the database
+    let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
+    builder.set_verify(SslVerifyMode::NONE);
+    let connector = MakeTlsConnector::new(builder.build());
+
+    let mut client = Client::connect(DB_URL, connector).unwrap();
+
+    // Gets the delta for a specific state group
+    let query_deltas = r#"
+        SELECT m.id, type, state_key, s.event_id
+        FROM state_groups AS m
+        LEFT JOIN state_groups_state AS s ON (m.id = s.state_group)
+        WHERE m.id = $1
+    "#;
+
+    // If there is no delta for that specific state group, then we still want to find
+    // the predecessor (so have split this into a different query)
+    let query_pred = r#"
+        SELECT prev_state_group
+        FROM state_group_edges 
+        WHERE state_group = $1
+    "#;
+
+    for (sg, entry) in state_group_map {
+        // get predecessor from state_group_edges
+        let mut pred_iter = client.query_raw(query_pred, &[sg]).unwrap();
+
+        // read out the predecessor value from the database
+        let database_pred = match pred_iter.next().unwrap() {
+            Some(p) => p.get(0),
+            None => None,
+        };
+
+        // if there was a predecessor then assert that it is unique
+        if database_pred.is_some() {
+            assert!(pred_iter.next().unwrap().is_none());
+        }
+
+        // check if it matches map
+        if database_pred != entry.prev_state_group {
+            println!(
+                "ERROR: predecessor for {} was {:?} (expected {:?})",
+                sg, database_pred, entry.prev_state_group
+            );
+            return false;
+        }
+        // needed so that can create another query
+        drop(pred_iter);
+
+        // Now check that deltas are the same
+        let mut state_map: StateMap<Atom> = StateMap::new();
+
+        // Get delta from state_groups_state
+        let mut rows = client.query_raw(query_deltas, &[sg]).unwrap();
+
+        while let Some(row) = rows.next().unwrap() {
+            // Copy the single delta from the predecessor stored in this row
+            if let Some(etype) = row.get::<_, Option<String>>(1) {
+                state_map.insert(
+                    &etype,
+                    &row.get::<_, String>(2),
+                    row.get::<_, String>(3).into(),
+                );
+            }
+        }
+
+        // Check that the delta matches the map
+        if state_map != entry.state_map {
+            println!("ERROR: delta for {} didn't match", sg);
+            println!("Expected: {:?}", entry.state_map);
+            println!("Actual: {:?}", state_map);
+            return false;
+        }
+    }
+    true
+}
+
+#[test]
+fn functions_are_self_consistent() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    for i in 0i64..=13i64 {
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        initial.insert(i, entry);
+
+        prev = Some(i)
+    }
+
+    empty_database();
+    add_contents_to_database("room1", &initial);
+
+    assert!(database_collapsed_states_match_map(&initial));
+    assert!(database_structure_matches_map(&initial));
+}

compressor_integration_tests/src/map_builder.rs

@@ -34,3 +34,129 @@ pub fn line_with_state(start: i64, end: i64) -> BTreeMap<i64, StateGroupEntry> {
 
     initial
 }
+
+/// Generates line segments in a chain of state groups each with state deltas
+///
+/// If called wiht start=0, end=13 this would build the following:
+///
+/// 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+///
+/// Where each group i has state:
+///     ('node','is',      i)
+///     ('group',  j, 'seen') - for all j less than i
+pub fn line_segments_with_state(start: i64, end: i64) -> BTreeMap<i64, StateGroupEntry> {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    for i in start..=end {
+        // if the state is a snapshot then set its predecessor to NONE
+        if (i - start) % 3 == 0 {
+            prev = None;
+        }
+
+        // create a blank entry for it
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+
+        // if it's a snapshot then add in all previous state
+        if prev.is_none() {
+            for j in start..i {
+                entry
+                    .state_map
+                    .insert("group", &j.to_string(), "seen".into());
+            }
+        }
+
+        // add in the new state for this state group
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        // put it into the initial map
+        initial.insert(i, entry);
+
+        // set this group as the predecessor for the next
+        prev = Some(i)
+    }
+    initial
+}
+
+/// This generates the correct compressed structure with 3,3 levels
+///
+/// Note: only correct structure when no impossible predecessors
+///
+/// Structure generated:
+///
+/// 0  3\      12
+/// 1  4 6\    13
+/// 2  5 7 9
+///     8 10
+///        11
+/// Where each group i has state:
+///     ('node','is',      i)
+///     ('group',  j, 'seen') - for all j less than i
+pub fn compressed_3_3_from_0_to_13_with_state() -> BTreeMap<i64, StateGroupEntry> {
+    let expected_edges: BTreeMap<i64, i64> = vec![
+        (1, 0),
+        (2, 1),
+        (4, 3),
+        (5, 4),
+        (6, 3),
+        (7, 6),
+        (8, 7),
+        (9, 6),
+        (10, 9),
+        (11, 10),
+        (13, 12),
+    ]
+    .into_iter()
+    .collect();
+
+    let mut expected: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    for i in 0i64..=13i64 {
+        let prev = expected_edges.get(&i);
+
+        //change from Option<&i64> to Option<i64>
+        let prev = prev.copied();
+
+        // create a blank entry for it
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+
+        // Add in all state between predecessor and now (non inclusive)
+        if let Some(p) = prev {
+            for j in (p + 1)..i {
+                entry
+                    .state_map
+                    .insert("group", &j.to_string(), "seen".into());
+            }
+        } else {
+            for j in 0i64..i {
+                entry
+                    .state_map
+                    .insert("group", &j.to_string(), "seen".into());
+            }
+        }
+
+        // add in the new state for this state group
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        // put it into the expected map
+        expected.insert(i, entry);
+    }
+    expected
+}