Skip to content

Commit d08d508

Browse files
ctronctron
committed
chore: add advisory to example
1 parent 6017201 commit d08d508

File tree

11 files changed

+221
-38
lines changed

11 files changed

+221
-38
lines changed

‎Cargo.lock‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎migration/Cargo.toml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,13 @@ trustify-module-storage = { workspace = true }
1717
anyhow = { workspace = true }
1818
bytes = { workspace = true }
1919
clap = { workspace = true, features = ["derive", "env"] }
20+
csaf = { workspace = true }
21+
cve = { workspace = true }
2022
futures = { workspace = true }
2123
futures-util = { workspace = true }
24+
humantime = { workspace = true }
2225
indicatif = { workspace = true, features = ["tokio", "futures"] }
26+
osv = { workspace = true, features = ["schema"] }
2327
sea-orm = { workspace = true }
2428
sea-orm-migration = { workspace = true, features = ["runtime-tokio-rustls", "sqlx-postgres", "with-uuid"] }
2529
serde-cyclonedx = { workspace = true }
@@ -29,7 +33,6 @@ tokio = { workspace = true, features = ["full"] }
2933
tracing = { workspace = true }
3034
tracing-subscriber = { workspace = true }
3135
uuid = { workspace = true, features = ["v5"] }
32-
humantime = { workspace = true }
3336

3437
[dev-dependencies]
3538
trustify-common = { workspace = true }

‎migration/src/data/document/advisory.rs‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
use super::Document;
2+
use bytes::Bytes;
3+
use sea_orm::prelude::*;
4+
use trustify_entity::advisory;
5+
use trustify_module_storage::service::StorageBackend;
6+
7+
#[allow(clippy::large_enum_variant)]
8+
pub enum Advisory {
9+
Cve(cve::Cve),
10+
Csaf(csaf::Csaf),
11+
Osv(osv::schema::Vulnerability),
12+
Other(Bytes),
13+
}
14+
15+
impl From<Bytes> for Advisory {
16+
fn from(value: Bytes) -> Self {
17+
serde_json::from_slice(&value)
18+
.map(Advisory::Cve)
19+
.or_else(|_| serde_json::from_slice(&value).map(Advisory::Csaf))
20+
.or_else(|_| serde_json::from_slice(&value).map(Advisory::Osv))
21+
.unwrap_or_else(|_err| Advisory::Other(value))
22+
}
23+
}
24+
25+
impl Document for Advisory {
26+
type Model = advisory::Model;
27+
28+
async fn all<C: ConnectionTrait>(tx: &C) -> Result<Vec<Self::Model>, DbErr> {
29+
advisory::Entity::find().all(tx).await
30+
}
31+
32+
async fn source<S, C>(model: &Self::Model, storage: &S, tx: &C) -> Result<Self, anyhow::Error>
33+
where
34+
S: StorageBackend + Send + Sync,
35+
C: ConnectionTrait,
36+
{
37+
super::load(model.source_document_id, storage, tx).await
38+
}
39+
}

‎migration/src/data/document/mod.rs‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,18 @@
1+
mod advisory;
2+
3+
pub use advisory::*;
4+
use anyhow::{anyhow, bail};
5+
use bytes::{Bytes, BytesMut};
6+
use futures_util::TryStreamExt;
17
mod sbom;
28
pub use sbom::*;
39

410
use crate::data::Partitionable;
5-
use sea_orm::{ConnectionTrait, DbErr};
6-
use trustify_module_storage::service::StorageBackend;
11+
use sea_orm::{ConnectionTrait, DbErr, EntityTrait};
12+
use trustify_common::id::Id;
13+
use trustify_entity::source_document;
14+
use trustify_module_storage::service::{StorageBackend, StorageKey};
15+
use uuid::Uuid;
716

817
#[allow(async_fn_in_trait)]
918
pub trait Document: Sized + Send + Sync {
@@ -18,3 +27,34 @@ pub trait Document: Sized + Send + Sync {
1827
S: StorageBackend + Send + Sync,
1928
C: ConnectionTrait;
2029
}
30+
31+
pub(crate) async fn load<D>(
32+
id: Uuid,
33+
storage: &(impl StorageBackend + Send + Sync),
34+
tx: &impl ConnectionTrait,
35+
) -> anyhow::Result<D>
36+
where
37+
D: Document + From<Bytes>,
38+
{
39+
let source = source_document::Entity::find_by_id(id).one(tx).await?;
40+
41+
let Some(source) = source else {
42+
bail!("Missing source document entry for: {id}");
43+
};
44+
45+
let stream = storage
46+
.retrieve(
47+
StorageKey::try_from(Id::Sha256(source.sha256))
48+
.map_err(|err| anyhow!("Invalid ID: {err}"))?,
49+
)
50+
.await
51+
.map_err(|err| anyhow!("Failed to retrieve document: {err}"))?
52+
.ok_or_else(|| anyhow!("Missing source document for: {id}"))?;
53+
54+
stream
55+
.try_collect::<BytesMut>()
56+
.await
57+
.map_err(|err| anyhow!("Failed to collect bytes: {err}"))
58+
.map(|bytes| bytes.freeze())
59+
.map(|bytes| bytes.into())
60+
}

‎migration/src/data/document/sbom.rs‎

Lines changed: 14 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,23 @@
11
use super::Document;
2-
use anyhow::{anyhow, bail};
3-
use bytes::BytesMut;
4-
use futures_util::TryStreamExt;
2+
use bytes::Bytes;
53
use sea_orm::prelude::*;
6-
use trustify_common::id::Id;
7-
use trustify_entity::{sbom, source_document};
8-
use trustify_module_storage::service::{StorageBackend, StorageKey};
4+
use trustify_entity::sbom;
5+
use trustify_module_storage::service::StorageBackend;
96

107
#[allow(clippy::large_enum_variant)]
118
pub enum Sbom {
129
CycloneDx(serde_cyclonedx::cyclonedx::v_1_6::CycloneDx),
1310
Spdx(spdx_rs::models::SPDX),
11+
Other(Bytes),
12+
}
13+
14+
impl From<Bytes> for Sbom {
15+
fn from(value: Bytes) -> Self {
16+
serde_json::from_slice(&value)
17+
.map(Sbom::Spdx)
18+
.or_else(|_| serde_json::from_slice(&value).map(Sbom::CycloneDx))
19+
.unwrap_or_else(|_err| Sbom::Other(value))
20+
}
1421
}
1522

1623
impl Document for Sbom {
@@ -25,31 +32,6 @@ impl Document for Sbom {
2532
S: StorageBackend + Send + Sync,
2633
C: ConnectionTrait,
2734
{
28-
let source = model.find_related(source_document::Entity).one(tx).await?;
29-
30-
let Some(source) = source else {
31-
bail!("Missing source document ID for SBOM: {}", model.sbom_id);
32-
};
33-
34-
let stream = storage
35-
.retrieve(
36-
StorageKey::try_from(Id::Sha256(source.sha256))
37-
.map_err(|err| anyhow!("Invalid ID: {err}"))?,
38-
)
39-
.await
40-
.map_err(|err| anyhow!("Failed to retrieve document: {err}"))?
41-
.ok_or_else(|| anyhow!("Missing source document for SBOM: {}", model.sbom_id))?;
42-
43-
stream
44-
.try_collect::<BytesMut>()
45-
.await
46-
.map_err(|err| anyhow!("Failed to collect bytes: {err}"))
47-
.map(|bytes| bytes.freeze())
48-
.and_then(|bytes| {
49-
serde_json::from_slice(&bytes)
50-
.map(Sbom::Spdx)
51-
.or_else(|_| serde_json::from_slice(&bytes).map(Sbom::CycloneDx))
52-
.map_err(|err| anyhow!("Failed to parse document: {err}"))
53-
})
35+
super::load(model.source_document_id, storage, tx).await
5436
}
5537
}

‎migration/src/data/migration.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ impl<'c> SchemaDataManager<'c> {
8080
}
8181
}
8282

83+
/// Run a data migration
8384
pub async fn process<D, N>(&self, name: &N, f: impl Handler<D>) -> Result<(), DbErr>
8485
where
8586
D: Document,

‎migration/src/data/mod.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,7 @@ impl<'c> DocumentProcessor for SchemaManager<'c> {
159159
}
160160
}
161161

162+
/// A handler for data migration of documents.
162163
#[macro_export]
163164
macro_rules! handler {
164165
(async | $doc:ident: $doc_ty:ty, $model:ident, $tx:ident | $body:block) => {{
@@ -179,13 +180,26 @@ macro_rules! handler {
179180
}};
180181
}
181182

183+
/// A handler for SBOMs.
184+
///
185+
/// See: [`handler!`].
182186
#[macro_export]
183187
macro_rules! sbom {
184188
(async | $doc:ident, $model:ident, $tx:ident | $body:block) => {
185189
$crate::handler!(async |$doc: $crate::data::Sbom, $model, $tx| $body)
186190
};
187191
}
188192

193+
/// A handler for advisories.
194+
///
195+
/// See: [`handler!`].
196+
#[macro_export]
197+
macro_rules! advisories {
198+
(async | $doc:ident, $model:ident, $tx:ident | $body:block) => {
199+
$crate::handler!(async |$doc: $crate::data::Advisory, $model, $tx| $body)
200+
};
201+
}
202+
189203
pub trait MigratorWithData {
190204
fn data_migrations() -> Vec<Box<dyn MigrationTraitWithData>>;
191205
}

‎migration/src/data/partition.rs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,20 @@ use std::{
33
hash::{DefaultHasher, Hash, Hasher},
44
num::NonZeroU64,
55
};
6-
use trustify_entity::sbom;
6+
use trustify_entity::{advisory, sbom};
77

88
#[derive(Debug, Copy, Clone)]
99
pub struct Partition {
1010
pub current: u64,
1111
pub total: NonZeroU64,
1212
}
1313

14+
/// A thing which can be distributed over different partitions via a hashed id.
15+
///
16+
/// The idea is that the thing returns a hash ID, which can then be distributed over partitions
17+
/// by using a "X of Y" approach. Where the thing is processed when "ID modulo Y == X".
1418
pub trait Partitionable {
19+
/// Get the hashed ID for the thing.
1520
fn hashed_id(&self) -> u64;
1621
}
1722

@@ -23,6 +28,14 @@ impl Partitionable for sbom::Model {
2328
}
2429
}
2530

31+
impl Partitionable for advisory::Model {
32+
fn hashed_id(&self) -> u64 {
33+
let mut hasher = DefaultHasher::new();
34+
self.id.hash(&mut hasher);
35+
hasher.finish()
36+
}
37+
}
38+
2639
impl Default for Partition {
2740
fn default() -> Self {
2841
Self::new_one()

‎migration/src/lib.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ mod m0001140_expand_spdx_licenses_function;
3434
mod m0001150_case_license_text_sbom_id_function;
3535
mod m0001160_improve_expand_spdx_licenses_function;
3636
mod m0001170_non_null_source_document_id;
37-
mod m0002000_example_data_migration;
37+
mod m0002000_example_sbom_data_migration;
38+
mod m0002010_example_advisory_data_migration;
3839

3940
pub struct Migrator;
4041

@@ -70,7 +71,8 @@ impl Migrator {
7071
.normal(m0001150_case_license_text_sbom_id_function::Migration)
7172
.normal(m0001160_improve_expand_spdx_licenses_function::Migration)
7273
.normal(m0001170_non_null_source_document_id::Migration)
73-
.data(m0002000_example_data_migration::Migration)
74+
.data(m0002000_example_sbom_data_migration::Migration)
75+
.data(m0002010_)
7476
}
7577
}
7678

migration/src/m0002000_example_data_migration.rs renamed to migration/src/m0002000_example_sbom_data_migration.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ impl MigrationTraitWithData for Migration {
4747
SbomDoc::Spdx(_sbom) => {
4848
model.properties = Set(serde_json::Value::Object(Default::default()));
4949
}
50+
SbomDoc::Other(_) => {
51+
// we ignore others
52+
}
5053
}
5154

5255
model.save(tx).await?;

0 commit comments

Comments
 (0)