Extract crates_io_database_dump crate

Author: Turbo87

Cargo.lock

@@ -51,6 +51,7 @@ bon = "=2.3.0"
 cargo-manifest = "=0.15.2"
 crates_io_cdn_logs = { path = "crates/crates_io_cdn_logs" }
 crates_io_database = { path = "crates/crates_io_database" }
+crates_io_database_dump = { path = "crates/crates_io_database_dump" }
 crates_io_env_vars = { path = "crates/crates_io_env_vars" }
 crates_io_github = { path = "crates/crates_io_github" }
 crates_io_index = { path = "crates/crates_io_index" }
@@ -116,7 +117,6 @@ tracing-subscriber = { version = "=0.3.18", features = ["env-filter", "json"] }
 typomania = { version = "=0.1.2", default-features = false }
 url = "=2.5.2"
 unicode-xid = "=0.2.6"
-zip = { version = "=2.2.0", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]
 bytes = "=1.8.0"
@@ -129,3 +129,4 @@ googletest = "=0.12.0"
 insta = { version = "=1.40.0", features = ["glob", "json", "redactions"] }
 regex = "=1.11.0"
 tokio = "=1.40.0"
+zip = { version = "=2.2.0", default-features = false, features = ["deflate"] }

Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "crates_io_database_dump"
+version = "0.0.0"
+license = "MIT OR Apache-2.0"
+edition = "2021"
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow = "=1.0.90"
+chrono = { version = "=0.4.38", default-features = false, features = ["clock", "serde"] }
+flate2 = "=1.0.34"
+minijinja = "=2.3.1"
+serde = { version = "=1.0.210", features = ["derive"] }
+serde_json = "=1.0.132"
+tar = "=0.4.42"
+tempfile = "=3.13.0"
+toml = "=0.8.19"
+tracing = "=0.1.40"
+zip = { version = "=2.2.0", default-features = false, features = ["deflate"] }
+
+[dev-dependencies]
+crates_io_test_db = { path = "../crates_io_test_db" }
+diesel = "=2.2.4"
+insta = { version = "=1.40.0", features = ["glob"] }

crates/crates_io_database_dump/Cargo.toml

@@ -1,10 +1,11 @@
+use serde::Deserialize;
 use std::collections::{BTreeMap, VecDeque};
 
 /// An enum indicating whether a column is included in the database dumps.
 /// Public columns are included, private are not.
 #[derive(Clone, Copy, Debug, Deserialize, PartialEq)]
 #[serde(rename_all = "lowercase")]
-pub(super) enum ColumnVisibility {
+pub enum ColumnVisibility {
     Private,
     Public,
 }
@@ -16,7 +17,7 @@ pub(super) enum ColumnVisibility {
 /// used in a `WHERE` clause to filter the rows of the table. The `columns`
 /// field maps column names to their respective visibilities.
 #[derive(Clone, Debug, Default, Deserialize)]
-pub(super) struct TableConfig {
+pub struct TableConfig {
     #[serde(default)]
     pub dependencies: Vec<String>,
     pub filter: Option<String>,
@@ -28,17 +29,17 @@ pub(super) struct TableConfig {
 /// Maps table names to the respective configurations. Used to load `dump_db.toml`.
 #[derive(Clone, Debug, Default, Deserialize)]
 #[serde(transparent)]
-pub(super) struct VisibilityConfig(pub BTreeMap<String, TableConfig>);
+pub struct VisibilityConfig(pub BTreeMap<String, TableConfig>);
 
 impl VisibilityConfig {
-    pub(super) fn get() -> Self {
+    pub fn get() -> Self {
         toml::from_str(include_str!("dump-db.toml")).unwrap()
     }
 
     /// Sort the tables in a way that dependencies come before dependent tables.
     ///
     /// Returns a vector of table names.
-    pub(super) fn topological_sort(&self) -> Vec<&str> {
+    pub fn topological_sort(&self) -> Vec<&str> {
         let mut num_deps = BTreeMap::new();
         let mut rev_deps: BTreeMap<_, Vec<_>> = BTreeMap::new();
         for (table, config) in self.0.iter() {

src/worker/jobs/dump_db/configuration.rs renamed to crates/crates_io_database_dump/src/configuration.rs

@@ -1,9 +1,8 @@
+use crate::configuration::{ColumnVisibility, TableConfig, VisibilityConfig};
 use anyhow::Context;
+use serde::Serialize;
 use std::{fs::File, path::Path};
-
-use crate::worker::jobs::dump_db::configuration::{
-    ColumnVisibility, TableConfig, VisibilityConfig,
-};
+use tracing::debug;
 
 pub fn gen_scripts(export_script: &Path, import_script: &Path) -> anyhow::Result<()> {
     let config = VisibilityConfig::get();
@@ -119,7 +118,7 @@ impl VisibilityConfig {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_util::test_db_connection;
+    use crates_io_test_db::TestDatabase;
     use diesel::prelude::*;
     use std::collections::HashSet;
     use std::iter::FromIterator;
@@ -128,8 +127,10 @@ mod tests {
     /// test database.
     #[test]
     fn check_visibility_config() {
-        let (_test_db, conn) = &mut test_db_connection();
-        let db_columns = HashSet::<Column>::from_iter(get_db_columns(conn));
+        let test_db = TestDatabase::new();
+        let mut conn = test_db.connect();
+
+        let db_columns = HashSet::<Column>::from_iter(get_db_columns(&mut conn));
         let vis_columns = VisibilityConfig::get()
             .0
             .iter()
@@ -167,6 +168,8 @@ mod tests {
     }
 
     mod information_schema {
+        use diesel::table;
+
         table! {
             information_schema.columns (table_schema, table_name, column_name) {
                 table_schema -> Text,