Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ import_dir = "/third-party"
admin = ["benjamin.747"]

# Set serveral root dirs in directory init
root_dirs = ["third-party", "project", "doc", "release", "model"]
root_dirs = ["third-party", "project", "doc", "release", "model", "toolchains"]

# git object storage type, support values can be "local_fs" or "s3"
storage_type = "s3"
Expand Down
2 changes: 1 addition & 1 deletion jupiter/src/storage/gpg_storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ impl GpgStorage {
#[test]
fn test_create_key() {
let key = "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQGNBGiGkcsBDADDQzGo993e+e/6h5lvYGtPt2kSHAmGIXyzeNUsePfEE2lewNLl\nuAnAUR56A5vxyV0zER1F8Sp2OGXola/x6yT86c0ZRQ6nItMojYTKJUfcy7o56F9Z\neL515XqFz5x29NXKfqaHc+EblqbvPIocC+uGEQD6l5nee6BDxmachUg+4SO8mqjd\nxmaGfpka0mmzQK2xgnFTsR0SkYXKmwf/w81vv5z53nXkJRUWUlZ0PHaCCaxO65fV\nvbLtaRVp7niRWnxmttNwG23AlIDDeSRaQ8FqJrCN3ZAdpfMoPmOZ1IWEmEb4p0Pn\n0vTz5WeT4kR9SmpMqbkpChWYaX8EgCpNrSqV62hrapVJ42fGb9nocuqSDNk7qrBY\n+EnzlPNbTSy9x0e7sbffvCrjxCfOnV6KmkPNBTs4un7cIThfyvZz5Aaw/BM6xT4v\n/01m7VLwT/+ZBKSP6GpRntsSnBitsUXtgN9URV+vnRMgMaXRjESIvWjeB+qMxBDU\nMhrN7eTQ11ByqsEAEQEAAbRLQWlkZW4gQ2hlbmcgKENvbnRhY3QgbXkgZW1haWwg\nZm9yIGFueSBxdWVzdGlvbnMpIDxjbi5haWRlbi5jaGVuZ0BnbWFpbC5jb20+iQHR\nBBMBCAA7FiEExZ7S27JTGFCk8+gpQv9AeDZzXb0FAmiGkcsCGwMFCwkIBwICIgIG\nFQoJCAsCBBYCAwECHgcCF4AACgkQQv9AeDZzXb0EGQv8C+1XkqXqLVmdWRKOhzJG\nXL7RB9Oexh2ueJYlojpyCFs8KXYGzIf/8L2SPxAmBh1ayEcDqvgUoYc/0lOl7pQr\n1rl7ZS2iGtYyxF9kyw0OEyLpXeQOXb1lUPG/k7S5xUq+xtsoByMhxJJDmW3fD99p\nLP1ApAW5P8jx4E3wdirxKb+5fip3CFvk0/pLzwCxxIf15ijG4nlWi/ZWIHo/VsMx\nGATOyL2Bn3BEaT95LEtvaEItyjnGp+bixqeZOlYFckQDG8nX7KQvZNQtJ9Ux9UJX\nDJ5OdSGwSv98EMMwmbAv9UGhANkgv+FAxAaQ1FCGHZD9PN+jVxlAVK4jDJeP4DOc\nBWU3SEWAVJqISa9ZmhBAU3mRJMYT6qqSl4r07tc1Ii8WYwFEnccjSi1axomnBln7\nDy01QggQNbLMAu/70HG28vDVtxBfe+WyAI/D59/uCnNO1phpQ7XVSN0D+dkleqba\nl6aRFd3Ll7EZDpW1kU+cmFbTFzo0ScUEdswT5pFO9JsguQGNBGiGkcsBDACVoLqH\nbM72FQP9delVRRY0UH0XbS7AmtzQ8wGZx2Wb3bHaY2H8WPJ4Zt/XWbplIy2sB9XC\nGcOTc5WXBOC74YQJ9Ub4o7+92S0ZFVaY7v3KGTXfSW4G+nghu0aS2RTXL2GctUJy\npSQmmX0yIR9vhFA65OtaG9QzY3vAXXtMWoCLYIIcOC/2b5F3KhEjK0YwDLCpBo2g\n7Xmu2o4kPY1Uuri1DGnfbMtJ1Ac2Mc2YodAlj9lapG2f5G2NV2TKwdYnCu8gCZ+z\nk+kaa9v7yn8O8j1kBEW5dmnaR0l0rGJXQcp6ffyzI0ulZB6f2u8lroAMf1j0oD45\nCS1kEZ/xL+N7WJ2JYWXHV4VdRZ3QOfDpSUzSh1wQ9Z1kVF8gHVsVtk5eSBsORFVp\npa1sVkba5eOIfOIPYqYwI+0qbKMe1SCLL88Hfhm1xoK7gQ1ErsxgqlAd8OP/SQ0y\ncqsyrVuyrrhlursQFW6Mo8o0aFKrJ1DWjgUX3By/pWp7n788ZD/dSYGOLs8AEQEA\nAYkBtgQYAQgAIBYhBMWe0tuyUxhQpPPoKUL/QHg2c129BQJohpHLAhsMAAoJEEL/\nQHg2c129fNoL/0OK3CBZgrvbzTXprRDc19AoDLfViIY0/nEAVITCvrTVMZXBD1Dx\nJN9cbvinjeZEUsoXsBHcbz2wUn1bhq/58e0ki1XmAC0ZJtJLFtbLAAvTJ2Wo56Os\nPNmE7OOV4VtHF4UWfRbkvg86oCjIY9+TS5v25GKIEkMZRFsNiVpC0uK5kNyaHeRB\nRqlG5ZV4pO12+EN66agJfRLRwlOmsyJ51/gFzdxP8Lygh2Br2WYU9girwxfQhUs/\nDeSsNHPkv1ESPvA2vDsKHLiatnp1gJfyC0vIgKUdG/v/7FzFtY2B3B0EbBD6iBZN\ncPVW3TGw8pbK+HT7vBhWjYxpk0evfVCUd67eXtsexutj30YszRcNn5ja+cfGBD+R\ndpDLm5hpuQtgfJYTuvwRtabRCZG8oRsOZfuRIkxWwN+VcjvmjWUF/1lSetAhpWs+\nVEG4kspCB7X0ePlBP1jPaOWzVphmV0e1eHo79qKS6038FySK81stvRux0DP57E3n\nF5MtAwnDBeT2Qg==\n=Q/C5\n-----END PGP PUBLIC KEY BLOCK-----\n";
let (pk, _) = SignedPublicKey::from_string(&key).expect("Failed to parse");
let (pk, _) = SignedPublicKey::from_string(key).expect("Failed to parse");

let key_id = format!("{:016X}", pk.key_id());
let fingerprint = format!("{:?}", pk.fingerprint()).to_uppercase();
Expand Down
106 changes: 95 additions & 11 deletions jupiter/src/utils/converter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,26 @@ pub fn init_trees(

inject_root_buck_files(&mut root_items, &mut blobs, &mono_config.root_dirs);

// Ensure the `toolchains` cell has a BUCK file at repo initialization time.
if let Some(toolchains_root_idx) = root_items.iter().position(|item| {
item.mode == TreeItemMode::Tree
&& item
.name
.trim_start_matches('/')
.trim_start_matches("./")
.trim_end_matches('/')
== "toolchains"
}) {
let toolchains_tree_id = root_items[toolchains_root_idx].id;
if let Some(toolchains_tree_idx) = trees.iter().position(|t| t.id == toolchains_tree_id) {
let mut toolchains_items = trees[toolchains_tree_idx].tree_items.clone();
inject_toolchains_buck_file(&mut toolchains_items, &mut blobs);
let toolchains_tree = Tree::from_tree_items(toolchains_items).unwrap();
trees[toolchains_tree_idx] = toolchains_tree.clone();
root_items[toolchains_root_idx].id = toolchains_tree.id;
}
}

let root = Tree::from_tree_items(root_items).unwrap();
(
trees.into_iter().map(|x| (x.id, x)).collect(),
Expand Down Expand Up @@ -474,6 +494,28 @@ fn inject_root_buck_files(
blobs.push(buckconfig_blob);
}

/// Injects a BUCK file into the toolchains directory.
fn inject_toolchains_buck_file(toolchains_items: &mut Vec<TreeItem>, blobs: &mut Vec<Blob>) {
let toolchains_content = generate_toolchains_buck_content();
let toolchains_blob = Blob::from_content(&toolchains_content);
toolchains_items.push(TreeItem {
mode: TreeItemMode::Blob,
id: toolchains_blob.id,
name: String::from("BUCK"),
});
blobs.push(toolchains_blob);
}

fn generate_toolchains_buck_content() -> String {
r#"load("@prelude//toolchains:demo.bzl", "system_demo_toolchains")

# All the default toolchains, suitable for a quick demo or early prototyping.
# Most real projects should copy/paste the implementation to configure them.
system_demo_toolchains()
"#
.to_string()
}

fn generate_buckroot_content() -> String {
// The .buckroot file is usually empty or contains a simple identifier.
String::new()
Expand All @@ -484,18 +526,57 @@ fn generate_buckroot_content() -> String {
const EXCLUDED_BUCK_CELL_DIRS: &[&str] = &["doc"];

fn generate_buckconfig_content(root_dirs: &[String]) -> String {
let cells = root_dirs
let normalized_root_dirs = root_dirs
.iter()
.map(|d| {
d.trim_start_matches('/')
.trim_start_matches("./")
.trim_end_matches('/')
.to_string()
})
.filter(|d| !EXCLUDED_BUCK_CELL_DIRS.contains(&d.as_str()))
.map(|d| format!(" {d} = {d}"))
.collect::<Vec<_>>()
.join("\n");

if cells.is_empty() {
"[cells]\n".to_string()
} else {
format!("[cells]\n{cells}\n")
.collect::<std::collections::HashSet<_>>();

let mut cells_lines = vec![" root = .".to_string(), " prelude = prelude".to_string()];

for dir in ["toolchains", "project", "release", "third-party"] {
if normalized_root_dirs.contains(dir) {
cells_lines.push(format!(" {dir} = {dir}"));
}
}

cells_lines.push(" buckal = toolchains/buckal-bundles".to_string());
cells_lines.push(" none = none".to_string());

let cells = cells_lines.join("\n");

format!(
r#"[cells]
{cells}

[cell_aliases]
config = prelude
ovr_config = prelude
fbcode = none
fbsource = none
fbcode_macros = none
buck = none

# Uses a copy of the prelude bundled with the buck2 binary. You can alternatively delete this
# section and vendor a copy of the prelude to the `prelude` directory of your project.
[external_cells]
prelude = bundled

[parser]
target_platform_detector_spec = target:root//...->prelude//platforms:default \
target:prelude//...->prelude//platforms:default \
target:toolchains//...->prelude//platforms:default

[build]
execution_platforms = prelude//platforms:default
default_target_platforms = prelude//platforms:default
"#
)
}

pub struct MegaModelConverter {
Expand Down Expand Up @@ -794,13 +875,16 @@ mod test {

#[test]
pub fn test_init_mega_dir() {
let mono_config = MonoConfig::default();
let mut mono_config = MonoConfig::default();
if !mono_config.root_dirs.iter().any(|d| d == "toolchains") {
mono_config.root_dirs.push("toolchains".to_string());
}
let converter = MegaModelConverter::init(&mono_config);
let mega_trees = converter.mega_trees.borrow().clone();
let mega_blobs = converter.mega_blobs.borrow().clone();
let dir_nums = mono_config.root_dirs.len();
assert_eq!(mega_trees.len(), dir_nums + 1);
assert_eq!(mega_blobs.len(), dir_nums + 2); // 2 = .buckconfig + .buckroot
assert_eq!(mega_blobs.len(), dir_nums + 3);
}

#[test]
Expand Down
24 changes: 24 additions & 0 deletions mono/src/server/http_server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,13 @@ pub fn remove_git_suffix(full_path: &str, git_suffix: &str) -> PathBuf {
PathBuf::from(full_path.replace(".git", "").replace(git_suffix, ""))
}

fn is_disallowed_root_repo_path(full_path: &str) -> bool {
matches!(
full_path.trim_start_matches('/').split('/').next(),
Some("third-party.git")
)
}

/// Spawns a background task to clean up expired Buck upload sessions.
///
/// Returns `None` if cleanup is disabled in configuration.
Expand Down Expand Up @@ -363,6 +370,11 @@ async fn handle_smart_protocol(
state: Arc<ProtocolApiState>,
) -> Result<Response, ProtocolError> {
let full_path = req.uri().path();
if is_disallowed_root_repo_path(full_path) {
return Err(ProtocolError::InvalidInput(
"Repository third-party.git is not supported".to_string(),
));
}
if full_path.ends_with("/info/refs") && req.method().eq(&Method::GET) {
let pack_protocol = SmartProtocol::new(
remove_git_suffix(full_path, "/info/refs"),
Expand Down Expand Up @@ -420,6 +432,18 @@ mod tests {

use super::*;

#[test]
fn test_disallow_third_party_git_root_repo() {
assert!(is_disallowed_root_repo_path("/third-party.git/info/refs"));
assert!(is_disallowed_root_repo_path(
"/third-party.git/git-receive-pack"
));
assert!(!is_disallowed_root_repo_path(
"/third-party/test.git/info/refs"
));
assert!(!is_disallowed_root_repo_path("/project.git/info/refs"));
}

#[test]
fn test_rewrite_lfs_uri_basic() {
let req = Request::builder()
Expand Down