zulip: sync stream membership

Author: davidtwco

docs/toml-schema.md

@@ -209,6 +209,37 @@ excluded-people = [
     "rylev",
 ]
 
+# Define the Zulip streams used by the team
+# It's optional, and there can be more than one.
+#
+# This will remove anyone who isn't in the team from the stream
+# so it should only be used for private streams at the moment.
+[[zulip-streams]]
+# The name of the Zulip stream (required)
+name = "t-overlords/private"
+# This can be set to false to avoid including all the team members in the stream
+# It's useful if you want to create the stream with a different set of members
+# It's optional, and the default is `true`.
+include-team-members = true
+# Include the following extra people in the Zulip stream. Their email address
+# or Zulip id will be fetched from their TOML in people/ (optional).
+extra-people = [
+    "alexcrichton",
+]
+# Include the following Zulip ids in the Zulip stream (optional).
+extra-zulip-ids = [
+    1234
+]
+# Include all the members of the following teams in the Zulip stream
+# (optional).
+extra-teams = [
+    "bots-nursery",
+]
+# Exclude the following people in the Zulip stream (optional).
+excluded-people = [
+    "rylev",
+]
+
 # Roles to define in Discord.
 [[discord-roles]]
 # The name of the role.

rust_team_data/src/v1.rs

@@ -126,6 +126,26 @@ pub struct ZulipGroups {
     pub groups: IndexMap<String, ZulipGroup>,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct ZulipStream {
+    pub name: String,
+    pub members: Vec<ZulipStreamMember>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum ZulipStreamMember {
+    // TODO(rylev): this variant can be removed once
+    // it is verified that no one is relying on it
+    Email(String),
+    Id(u64),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub struct ZulipStreams {
+    pub streams: IndexMap<String, ZulipStream>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct Permission {
     pub people: Vec<PermissionPerson>,

src/data.rs

@@ -1,4 +1,4 @@
-use crate::schema::{Config, List, Person, Repo, Team, ZulipGroup};
+use crate::schema::{Config, List, Person, Repo, Team, ZulipGroup, ZulipStream};
 use anyhow::{bail, Context as _, Error};
 use serde::de::DeserializeOwned;
 use std::collections::{HashMap, HashSet};
@@ -140,6 +140,16 @@ impl Data {
         Ok(groups)
     }
 
+    pub(crate) fn zulip_streams(&self) -> Result<HashMap<String, ZulipStream>, Error> {
+        let mut streams = HashMap::new();
+        for team in self.teams() {
+            for stream in team.zulip_streams(self)? {
+                streams.insert(stream.name().to_string(), stream);
+            }
+        }
+        Ok(streams)
+    }
+
     pub(crate) fn team(&self, name: &str) -> Option<&Team> {
         self.teams.get(name)
     }

src/schema.rs

@@ -179,6 +179,8 @@ pub(crate) struct Team {
     lists: Vec<TeamList>,
     #[serde(default)]
     zulip_groups: Vec<RawZulipGroup>,
+    #[serde(default)]
+    zulip_streams: Vec<RawZulipStream>,
     discord_roles: Option<Vec<DiscordRole>>,
 }
 
@@ -421,6 +423,60 @@ impl Team {
         Ok(groups)
     }
 
+    pub(crate) fn raw_zulip_streams(&self) -> &[RawZulipStream] {
+        &self.zulip_streams
+    }
+
+    pub(crate) fn zulip_streams(&self, data: &Data) -> Result<Vec<ZulipStream>, Error> {
+        let mut streams = Vec::new();
+        let zulip_streams = self.raw_zulip_streams();
+
+        for raw_stream in zulip_streams {
+            let mut stream = ZulipStream {
+                name: raw_stream.name.clone(),
+                members: Vec::new(),
+            };
+
+            let mut members = if raw_stream.include_team_members {
+                self.members(data)?
+            } else {
+                HashSet::new()
+            };
+            for person in &raw_stream.extra_people {
+                members.insert(person.as_str());
+            }
+            for team in &raw_stream.extra_teams {
+                let team = data
+                    .team(team)
+                    .ok_or_else(|| format_err!("team {} is missing", team))?;
+                members.extend(team.members(data)?);
+            }
+            for excluded in &raw_stream.excluded_people {
+                if !members.remove(excluded.as_str()) {
+                    bail!("'{excluded}' was specifically excluded from the Zulip stream '{}' but they were already not included", raw_stream.name);
+                }
+            }
+
+            for member in members.iter() {
+                let member = data.person(member).ok_or_else(|| {
+                    format_err!("{} does not have a person configuration", member)
+                })?;
+                let member = match (member.github.clone(), member.zulip_id) {
+                    (github, Some(zulip_id)) => {
+                        ZulipStreamMember::MemberWithId { github, zulip_id }
+                    }
+                    (github, _) => ZulipStreamMember::MemberWithoutId { github },
+                };
+                stream.members.push(member);
+            }
+            for &extra in &raw_stream.extra_zulip_ids {
+                stream.members.push(ZulipStreamMember::JustId(extra));
+            }
+            streams.push(stream);
+        }
+        Ok(streams)
+    }
+
     pub(crate) fn permissions(&self) -> &Permissions {
         &self.permissions
     }
@@ -691,6 +747,22 @@ pub(crate) struct RawZulipGroup {
     pub(crate) excluded_people: Vec<String>,
 }
 
+#[derive(serde_derive::Deserialize, Debug)]
+#[serde(rename_all = "kebab-case", deny_unknown_fields)]
+pub(crate) struct RawZulipStream {
+    pub(crate) name: String,
+    #[serde(default = "default_true")]
+    pub(crate) include_team_members: bool,
+    #[serde(default)]
+    pub(crate) extra_people: Vec<String>,
+    #[serde(default)]
+    pub(crate) extra_zulip_ids: Vec<u64>,
+    #[serde(default)]
+    pub(crate) extra_teams: Vec<String>,
+    #[serde(default)]
+    pub(crate) excluded_people: Vec<String>,
+}
+
 #[derive(Debug)]
 pub(crate) struct List {
     address: String,
@@ -736,6 +808,29 @@ pub(crate) enum ZulipGroupMember {
     MemberWithoutId { github: String },
 }
 
+#[derive(Debug)]
+pub(crate) struct ZulipStream {
+    name: String,
+    members: Vec<ZulipStreamMember>,
+}
+
+impl ZulipStream {
+    pub(crate) fn name(&self) -> &str {
+        &self.name
+    }
+
+    pub(crate) fn members(&self) -> &[ZulipStreamMember] {
+        &self.members
+    }
+}
+
+#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq, Hash)]
+pub(crate) enum ZulipStreamMember {
+    MemberWithId { github: String, zulip_id: u64 },
+    JustId(u64),
+    MemberWithoutId { github: String },
+}
+
 fn default_true() -> bool {
     true
 }

src/static_api.rs

@@ -1,6 +1,7 @@
 use crate::data::Data;
 use crate::schema::{
     Bot, Email, MergeBot, Permissions, RepoPermission, TeamKind, ZulipGroupMember,
+    ZulipStreamMember,
 };
 use anyhow::{ensure, Context as _, Error};
 use indexmap::IndexMap;
@@ -30,6 +31,7 @@ impl<'a> Generator<'a> {
         self.generate_repos()?;
         self.generate_lists()?;
         self.generate_zulip_groups()?;
+        self.generate_zulip_streams()?;
         self.generate_permissions()?;
         self.generate_rfcbot()?;
         self.generate_zulip_map()?;
@@ -303,6 +305,37 @@ impl<'a> Generator<'a> {
         Ok(())
     }
 
+    fn generate_zulip_streams(&self) -> Result<(), Error> {
+        let mut streams = IndexMap::new();
+
+        for stream in self.data.zulip_streams()?.values() {
+            let mut members = stream.members().to_vec();
+            members.sort();
+            streams.insert(
+                stream.name().to_string(),
+                v1::ZulipStream {
+                    name: stream.name().to_string(),
+                    members: members
+                        .into_iter()
+                        .filter_map(|m| match m {
+                            ZulipStreamMember::MemberWithId { zulip_id, .. } => {
+                                Some(v1::ZulipStreamMember::Id(zulip_id))
+                            }
+                            ZulipStreamMember::JustId(zulip_id) => {
+                                Some(v1::ZulipStreamMember::Id(zulip_id))
+                            }
+                            ZulipStreamMember::MemberWithoutId { .. } => None,
+                        })
+                        .collect(),
+                },
+            );
+        }
+
+        streams.sort_keys();
+        self.add("v1/zulip-streams.json", &v1::ZulipStreams { streams })?;
+        Ok(())
+    }
+
     fn generate_permissions(&self) -> Result<(), Error> {
         for perm in &Permissions::available(self.data.config()) {
             let allowed = crate::permissions::allowed_people(self.data, perm)?;

tests/static-api/_expected/v1/zulip-streams.json

@@ -0,0 +1,15 @@
+{
+  "streams": {
+    "t-foo/private": {
+      "name": "t-foo/private",
+      "members": [
+        {
+          "id": 1234
+        },
+        {
+          "id": 4321
+        }
+      ]
+    }
+  }
+}

tests/static-api/teams/foo.toml

@@ -49,3 +49,6 @@ extra-teams = ["wg-test"]
 
 [[zulip-groups]]
 name = "T-foo"
+
+[[zulip-streams]]
+name = "t-foo/private"