diff --git a/src/api/data_types/chunking/upload/capability.rs b/src/api/data_types/chunking/upload/capability.rs index c3c7a3f94a..03c71dacc6 100644 --- a/src/api/data_types/chunking/upload/capability.rs +++ b/src/api/data_types/chunking/upload/capability.rs @@ -2,9 +2,6 @@ use serde::{Deserialize, Deserializer}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ChunkUploadCapability { - /// Chunked upload of release files - ReleaseFiles, - /// Chunked upload of standalone artifact bundles ArtifactBundles, @@ -31,7 +28,6 @@ impl<'de> Deserialize<'de> for ChunkUploadCapability { D: Deserializer<'de>, { Ok(match String::deserialize(deserializer)?.as_str() { - "release_files" => ChunkUploadCapability::ReleaseFiles, "artifact_bundles" => ChunkUploadCapability::ArtifactBundles, "artifact_bundles_v2" => ChunkUploadCapability::ArtifactBundlesV2, "dartsymbolmap" => ChunkUploadCapability::DartSymbolMap, diff --git a/src/api/mod.rs b/src/api/mod.rs index c5a35f961e..c9f2a0e9d6 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -14,10 +14,8 @@ mod pagination; use std::borrow::Cow; use std::cell::RefCell; use std::collections::HashMap; -use std::ffi::OsStr; use std::fs::File; use std::io::{self, Read as _, Write}; -use std::path::Path; use std::rc::Rc; use std::sync::Arc; use std::{fmt, thread}; @@ -46,8 +44,7 @@ use uuid::Uuid; use crate::api::errors::{ProjectRenamedError, RetryError}; use crate::config::{Auth, Config}; -use crate::constants::{ARCH, DEFAULT_URL, EXT, PLATFORM, RELEASE_REGISTRY_LATEST_URL, VERSION}; -use crate::utils::file_upload::LegacyUploadContext; +use crate::constants::{ARCH, EXT, PLATFORM, RELEASE_REGISTRY_LATEST_URL, VERSION}; use crate::utils::http::{self, is_absolute_url}; use crate::utils::non_empty::NonEmptySlice; use crate::utils::progress::{ProgressBar, ProgressBarMode}; @@ -87,11 +84,6 @@ pub struct AuthenticatedApi<'a> { api: &'a Api, } -pub struct RegionSpecificApi<'a> { - api: &'a AuthenticatedApi<'a>, - region_url: Option>, -} - /// Represents an HTTP method that is used by the API. #[derive(Eq, PartialEq, Debug)] pub enum Method { @@ -446,7 +438,7 @@ impl Api { } } -impl<'a> AuthenticatedApi<'a> { +impl AuthenticatedApi<'_> { // Pass through low-level methods to API. /// Convenience method to call self.api.get. @@ -482,110 +474,6 @@ impl<'a> AuthenticatedApi<'a> { self.get("/")?.convert() } - /// Lists release files for the given `release`, filtered by a set of checksums. - /// When empty checksums list is provided, fetches all possible artifacts. - pub fn list_release_files_by_checksum( - &self, - org: &str, - project: Option<&str>, - release: &str, - checksums: &[String], - ) -> ApiResult> { - let mut rv = vec![]; - let mut cursor = "".to_owned(); - loop { - let mut path = if let Some(project) = project { - format!( - "/projects/{}/{}/releases/{}/files/?cursor={}", - PathArg(org), - PathArg(project), - PathArg(release), - QueryArg(&cursor), - ) - } else { - format!( - "/organizations/{}/releases/{}/files/?cursor={}", - PathArg(org), - PathArg(release), - QueryArg(&cursor), - ) - }; - - let mut checksums_qs = String::new(); - for checksum in checksums.iter() { - checksums_qs.push_str(&format!("&checksum={}", QueryArg(checksum))); - } - // We have a 16kb buffer for reach request configured in nginx, - // so do not even bother trying if it's too long. - // (16_384 limit still leaves us with 384 bytes for the url itself). - if !checksums_qs.is_empty() && checksums_qs.len() <= 16_000 { - path.push_str(&checksums_qs); - } - - let resp = self.get(&path)?; - if resp.status() == 404 || (resp.status() == 400 && !cursor.is_empty()) { - if rv.is_empty() { - return Err(ApiErrorKind::ReleaseNotFound.into()); - } else { - break; - } - } - - let pagination = resp.pagination(); - rv.extend(resp.convert::>()?); - if let Some(next) = pagination.into_next_cursor() { - cursor = next; - } else { - break; - } - } - Ok(rv) - } - - /// Lists all the release files for the given `release`. - pub fn list_release_files( - &self, - org: &str, - project: Option<&str>, - release: &str, - ) -> ApiResult> { - self.list_release_files_by_checksum(org, project, release, &[]) - } - - /// Deletes a single release file. Returns `true` if the file was - /// deleted or `false` otherwise. - pub fn delete_release_file( - &self, - org: &str, - project: Option<&str>, - version: &str, - file_id: &str, - ) -> ApiResult { - let path = if let Some(project) = project { - format!( - "/projects/{}/{}/releases/{}/files/{}/", - PathArg(org), - PathArg(project), - PathArg(version), - PathArg(file_id) - ) - } else { - format!( - "/organizations/{}/releases/{}/files/{}/", - PathArg(org), - PathArg(version), - PathArg(file_id) - ) - }; - - let resp = self.delete(&path)?; - if resp.status() == 404 { - Ok(false) - } else { - resp.into_result().map(|_| true) - } - } - /// Creates a new release. pub fn new_release(&self, org: &str, release: &NewRelease) -> ApiResult { // for single project releases use the legacy endpoint that is project bound. @@ -1181,50 +1069,6 @@ impl<'a> AuthenticatedApi<'a> { } Ok(rv) } - - fn get_region_url(&self, org: &str) -> ApiResult { - self.get(&format!("/organizations/{org}/region/")) - .and_then(|resp| resp.convert::()) - .map(|region| region.url) - } - - pub fn region_specific(&'a self, org: &'a str) -> RegionSpecificApi<'a> { - let base_url = self.api.config.get_base_url(); - if base_url.is_err() - || base_url.expect("base_url should not be error") != DEFAULT_URL.trim_end_matches('/') - { - // Do not specify a region URL unless the URL is configured to https://sentry.io (i.e. the default). - return RegionSpecificApi { - api: self, - region_url: None, - }; - } - - let region_url = match self - .api - .config - .get_auth() - .expect("auth should not be None for authenticated API!") - { - Auth::Token(token) => match token.payload() { - Some(payload) => Some(payload.region_url.clone().into()), - None => { - let region_url = self.get_region_url(org); - if let Err(err) = ®ion_url { - log::warn!("Failed to get region URL due to following error: {err}"); - log::info!("Failling back to the default region."); - } - - region_url.ok().map(|url| url.into()) - } - }, - }; - - RegionSpecificApi { - api: self, - region_url, - } - } } /// Available datasets for fetching organization events @@ -1298,73 +1142,6 @@ impl FetchEventsOptions<'_> { } } -impl RegionSpecificApi<'_> { - fn request(&self, method: Method, url: &str) -> ApiResult { - self.api - .api - .request(method, url, self.region_url.as_deref()) - } - - /// Uploads a new release file. The file is loaded directly from the file - /// system and uploaded as `name`. - pub fn upload_release_file( - &self, - context: &LegacyUploadContext, - contents: &[u8], - name: &str, - headers: Option<&[(String, String)]>, - progress_bar_mode: ProgressBarMode, - ) -> ApiResult> { - let path = if let Some(project) = context.project() { - format!( - "/projects/{}/{}/releases/{}/files/", - PathArg(context.org()), - PathArg(project), - PathArg(context.release()) - ) - } else { - format!( - "/organizations/{}/releases/{}/files/", - PathArg(context.org()), - PathArg(context.release()) - ) - }; - - let mut form = curl::easy::Form::new(); - - let filename = Path::new(name) - .file_name() - .and_then(OsStr::to_str) - .unwrap_or("unknown.bin"); - form.part("file") - .buffer(filename, contents.to_vec()) - .add()?; - form.part("name").contents(name.as_bytes()).add()?; - if let Some(dist) = context.dist() { - form.part("dist").contents(dist.as_bytes()).add()?; - } - - if let Some(headers) = headers { - for (key, value) in headers { - form.part("header") - .contents(format!("{key}:{value}").as_bytes()) - .add()?; - } - } - - let resp = self - .request(Method::Post, &path)? - .with_form_data(form)? - .progress_bar_mode(progress_bar_mode) - .send()?; - if resp.status() == 409 { - Ok(None) - } else { - resp.convert_rnf(ApiErrorKind::ReleaseNotFound) - } - } -} - fn send_req( handle: &mut curl::easy::Easy, out: &mut W, @@ -1858,14 +1635,6 @@ pub struct AuthInfo { pub user: Option, } -/// A release artifact -#[derive(Clone, Deserialize, Debug)] -pub struct Artifact { - pub id: String, - pub name: String, - pub dist: Option, -} - /// Information for new releases #[derive(Debug, Serialize, Default)] pub struct NewRelease { diff --git a/src/utils/auth_token/org_auth_token.rs b/src/utils/auth_token/org_auth_token.rs index faa596a23a..c464a2d0cf 100644 --- a/src/utils/auth_token/org_auth_token.rs +++ b/src/utils/auth_token/org_auth_token.rs @@ -14,7 +14,6 @@ pub struct OrgAuthToken { /// Represents the payload data of an org auth token. #[derive(Clone, Debug, Deserialize)] pub struct AuthTokenPayload { - pub region_url: String, pub org: String, // URL may be missing from some old auth tokens, see getsentry/sentry#57123 diff --git a/src/utils/file_upload.rs b/src/utils/file_upload.rs index 7990d11203..fa24f81c14 100644 --- a/src/utils/file_upload.rs +++ b/src/utils/file_upload.rs @@ -1,7 +1,7 @@ //! Searches, processes and uploads release files. -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use std::ffi::OsStr; -use std::fmt::{self, Display}; +use std::fmt; use std::path::PathBuf; use std::str; use std::sync::Arc; @@ -9,14 +9,10 @@ use std::time::{Duration, Instant}; use anyhow::{anyhow, bail, Result}; use console::style; -use parking_lot::RwLock; -use rayon::prelude::*; -use rayon::ThreadPoolBuilder; use sha1_smol::Digest; use symbolic::common::ByteView; use symbolic::debuginfo::js; use symbolic::debuginfo::sourcebundle::SourceFileType; -use thiserror::Error; use crate::api::NewRelease; use crate::api::{Api, ChunkServerOptions, ChunkUploadCapability}; @@ -24,7 +20,7 @@ use crate::constants::DEFAULT_MAX_WAIT; use crate::utils::chunks::{upload_chunks, Chunk, ASSEMBLE_POLL_INTERVAL}; use crate::utils::fs::get_sha1_checksums; use crate::utils::non_empty::NonEmptySlice; -use crate::utils::progress::{ProgressBar, ProgressBarMode, ProgressStyle}; +use crate::utils::progress::{ProgressBar, ProgressStyle}; use crate::utils::source_bundle; use super::file_search::ReleaseFileMatch; @@ -81,112 +77,6 @@ impl UploadContext<'_> { } } -#[derive(Debug, Error)] -pub enum LegacyUploadContextError { - #[error("a release is required for this upload")] - ReleaseMissing, - #[error("only a single project is supported for this upload")] - ProjectMultiple, -} - -/// Represents the context for legacy release uploads. -/// -/// `LegacyUploadContext` contains information needed for legacy (non-chunked) -/// uploads. Legacy uploads are primarily used when uploading to old self-hosted -/// Sentry servers, which do not support receiving chunked uploads. -/// -/// Unlike chunked uploads, legacy uploads require a release to be set, -/// and do not need to have chunk-upload-related fields. -#[derive(Debug, Default)] -pub struct LegacyUploadContext<'a> { - org: &'a str, - project: Option<&'a str>, - release: &'a str, - dist: Option<&'a str>, -} - -impl LegacyUploadContext<'_> { - pub fn org(&self) -> &str { - self.org - } - - pub fn project(&self) -> Option<&str> { - self.project - } - - pub fn release(&self) -> &str { - self.release - } - - pub fn dist(&self) -> Option<&str> { - self.dist - } -} - -impl Display for LegacyUploadContext<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - writeln!( - f, - "{} {}", - style("> Organization:").dim(), - style(self.org).yellow() - )?; - writeln!( - f, - "{} {}", - style("> Project:").dim(), - style(self.project.unwrap_or("None")).yellow() - )?; - writeln!( - f, - "{} {}", - style("> Release:").dim(), - style(self.release).yellow() - )?; - writeln!( - f, - "{} {}", - style("> Dist:").dim(), - style(self.dist.unwrap_or("None")).yellow() - )?; - write!( - f, - "{} {}", - style("> Upload type:").dim(), - style("single file/legacy upload").yellow() - ) - } -} - -impl<'a> TryFrom<&'a UploadContext<'_>> for LegacyUploadContext<'a> { - type Error = LegacyUploadContextError; - - fn try_from(value: &'a UploadContext) -> Result { - let &UploadContext { - org, - projects, - release, - dist, - .. - } = value; - - let project = Some(match <&[_]>::from(projects) { - [] => unreachable!("NonEmptySlice cannot be empty"), - [project] => Ok(project.as_str()), - [_, _, ..] => Err(LegacyUploadContextError::ProjectMultiple), - }?); - - let release = release.ok_or(LegacyUploadContextError::ReleaseMissing)?; - - Ok(Self { - org, - project, - release, - dist, - }) - } -} - #[derive(Eq, PartialEq, Debug, Copy, Clone, Hash)] pub enum LogLevel { Warning, @@ -339,150 +229,10 @@ impl<'a> FileUpload<'a> { pub fn upload(&self) -> Result<()> { // multiple projects OK initialize_legacy_release_upload(self.context)?; - - if self - .context - .chunk_upload_options - .supports(ChunkUploadCapability::ReleaseFiles) - { - // multiple projects OK - return upload_files_chunked( - self.context, - &self.files, - self.context.chunk_upload_options, - ); - } - - log::warn!( - "[DEPRECATION NOTICE] Your Sentry server does not support chunked uploads for \ - sourcemaps/release files. Falling back to deprecated upload method, which has fewer \ - features and is less reliable. Support for this deprecated upload method will be \ - removed in Sentry CLI 3.0.0. Please upgrade your Sentry server, or if you cannot \ - upgrade, pin your Sentry CLI version to 2.x, so you don't get upgraded to 3.x when \ - it is released." - ); - - // Do not permit uploads of more than 20k files if the server does not - // support artifact bundles. This is a temporary downside protection to - // protect users from uploading more sources than we support. - if self.files.len() > 20_000 { - bail!( - "Too many sources: {} exceeds maximum allowed files per release", - &self.files.len() - ); - } - - let concurrency = self.context.chunk_upload_options.concurrency as usize; - - let legacy_context = &self.context.try_into().map_err(|e| { - anyhow::anyhow!( - "Error while performing legacy upload: {e}. \ - If you would like to upload files {}, you need to upgrade your Sentry server \ - or switch to our SaaS offering.", - match e { - LegacyUploadContextError::ReleaseMissing => "without specifying a release", - LegacyUploadContextError::ProjectMultiple => - "to multiple projects simultaneously", - } - ) - })?; - - #[expect(deprecated, reason = "fallback to legacy upload")] - upload_files_parallel(legacy_context, &self.files, concurrency) + upload_files_chunked(self.context, &self.files, self.context.chunk_upload_options) } } -#[deprecated = "this non-chunked upload mechanism is deprecated in favor of upload_files_chunked"] -fn upload_files_parallel( - context: &LegacyUploadContext, - files: &SourceFiles, - num_threads: usize, -) -> Result<()> { - let api = Api::current(); - let release = context.release(); - - // get a list of release files first so we know the file IDs of - // files that already exist. - let release_files: HashMap<_, _> = api - .authenticated()? - .list_release_files(context.org, context.project(), release)? - .into_iter() - .map(|artifact| ((artifact.dist, artifact.name), artifact.id)) - .collect(); - - println!( - "{} Uploading source maps for release {}", - style(">").dim(), - style(release).cyan() - ); - - let progress_style = ProgressStyle::default_bar().template(&format!( - "{} Uploading {} source map{}...\ - \n{{wide_bar}} {{bytes}}/{{total_bytes}} ({{eta}})", - style(">").dim(), - style(files.len().to_string()).yellow(), - if files.len() == 1 { "" } else { "s" } - )); - - let total_bytes = files.values().map(|file| file.contents.len()).sum(); - let files = files.iter().collect::>(); - - let pb = Arc::new(ProgressBar::new(total_bytes)); - pb.set_style(progress_style); - - let pool = ThreadPoolBuilder::new().num_threads(num_threads).build()?; - let bytes = Arc::new(RwLock::new(vec![0u64; files.len()])); - - pool.install(|| { - files - .into_par_iter() - .enumerate() - .map(|(index, (_, file))| -> Result<()> { - let api = Api::current(); - let authenticated_api = api.authenticated()?; - let mode = ProgressBarMode::Shared(( - pb.clone(), - file.contents.len() as u64, - index, - bytes.clone(), - )); - - if let Some(old_id) = - release_files.get(&(context.dist.map(|x| x.into()), file.url.clone())) - { - authenticated_api - .delete_release_file(context.org, context.project, release, old_id) - .ok(); - } - - authenticated_api - .region_specific(context.org) - .upload_release_file( - context, - &file.contents, - &file.url, - Some( - file.headers - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect::>() - .as_slice(), - ), - mode, - )?; - - Ok(()) - }) - .collect::>() - })?; - - pb.finish_and_clear(); - - println!("{context}"); - - Ok(()) -} - fn poll_assemble( checksum: Digest, chunks: &[Digest],