From f4a020f5c5228be2e1d87b6404d2e57733946c45 Mon Sep 17 00:00:00 2001 From: Josh Holmer Date: Fri, 27 May 2022 04:10:58 -0400 Subject: [PATCH 1/2] Implement building from sources --- .github/workflows/aom.yml | 70 +++++++++-------- aom-sys/Cargo.toml | 1 + aom-sys/build.rs | 157 +++++++++++++++++++++++++++++++++++++- aom-sys/data/accounting.h | 79 +++++++++++++++++++ aom-sys/data/inspection.h | 85 +++++++++++++++++++++ 5 files changed, 356 insertions(+), 36 deletions(-) create mode 100644 aom-sys/data/accounting.h create mode 100644 aom-sys/data/inspection.h diff --git a/.github/workflows/aom.yml b/.github/workflows/aom.yml index 8aeefd5..76d7807 100644 --- a/.github/workflows/aom.yml +++ b/.github/workflows/aom.yml @@ -4,39 +4,43 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - name: Install nasm - uses: ilammy/setup-nasm@v1 - - - name: Install Rust stable - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - - name: Install aom - run: | - git clone --depth 1 https://aomedia.googlesource.com/aom - cd aom - mkdir -p build - cd build - cmake -DCMAKE_INSTALL_PREFIX=$HOME/aom_dir \ - -DBUILD_SHARED_LIBS=1 \ - -DENABLE_TESTS=0 \ - -DENABLE_EXAMPLES=0 \ - .. - make -j12 - make install - - - name: Run tests - run: | - export PKG_CONFIG_PATH=$HOME/aom_dir/lib/pkgconfig:$PKG_CONFIG_PATH - export LD_LIBRARY_PATH=$HOME/aom_dir/lib:$LD_LIBRARY_PATH - cargo test --all-features --verbose - cargo doc --all-features --verbose + - uses: actions/checkout@v2 + + - name: Install nasm + uses: ilammy/setup-nasm@v1 + + - name: Install Rust stable + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - name: Install aom + run: | + git clone --depth 1 https://aomedia.googlesource.com/aom + cd aom + mkdir -p build + cd build + cmake -DCMAKE_INSTALL_PREFIX=$HOME/aom_dir \ + -DBUILD_SHARED_LIBS=1 \ + -DENABLE_TESTS=0 \ + -DENABLE_EXAMPLES=0 \ + .. + make -j12 + make install + + - name: Test against system library + run: | + export PKG_CONFIG_PATH=$HOME/aom_dir/lib/pkgconfig:$PKG_CONFIG_PATH + export LD_LIBRARY_PATH=$HOME/aom_dir/lib:$LD_LIBRARY_PATH + RUST_BACKTRACE=1 cargo test --verbose + RUST_BACKTRACE=1 cargo doc --verbose + + - name: Test using built sources + run: | + RUST_BACKTRACE=1 cargo test --all-features --verbose + RUST_BACKTRACE=1 cargo doc --all-features --verbose diff --git a/aom-sys/Cargo.toml b/aom-sys/Cargo.toml index 4326ce1..1ab2fb8 100644 --- a/aom-sys/Cargo.toml +++ b/aom-sys/Cargo.toml @@ -13,6 +13,7 @@ aom = "3.0.0" [features] build_sources = [] +accounting = ["build_sources"] [build-dependencies] bindgen = "0.59.1" diff --git a/aom-sys/build.rs b/aom-sys/build.rs index 149ce46..7c028b7 100644 --- a/aom-sys/build.rs +++ b/aom-sys/build.rs @@ -1,7 +1,9 @@ -use std::env; -use std::fs::File; +use std::fs::{self, File}; use std::io::Write; use std::path::PathBuf; +use std::process::Command; +use std::thread::available_parallelism; +use std::{env, io}; fn format_write(builder: bindgen::Builder) -> String { builder @@ -13,7 +15,31 @@ fn format_write(builder: bindgen::Builder) -> String { } fn main() { - let libs = system_deps::Config::new().probe().unwrap(); + let libs = if env::var("CARGO_FEATURE_BUILD_SOURCES").is_ok() { + println!( + "cargo:rustc-link-search=native={}", + search().join("lib").to_string_lossy() + ); + println!("cargo:rustc-link-lib=static=aom"); + if fs::metadata(&search().join("lib").join("libaom.a")).is_err() { + fs::create_dir_all(&output()).expect("failed to create build directory"); + fetch().unwrap(); + build().unwrap(); + } + + env::set_var("SYSTEM_DEPS_LINK", "static"); + env::set_var("SYSTEM_DEPS_BUILD_INTERNAL", "always"); + system_deps::Config::new() + .add_build_internal("aom", |lib, version| { + // Actually build the library here + system_deps::Library::from_internal_pkg_config(pkg_config(), lib, version) + }) + .probe() + .unwrap() + } else { + // Use system libraries + system_deps::Config::new().probe().unwrap() + }; let headers = libs.all_include_paths(); let mut builder = bindgen::builder() @@ -21,6 +47,11 @@ fn main() { .blocklist_type("max_align_t") .size_t_is_usize(true) .default_enum_style(bindgen::EnumVariation::ModuleConsts); + if env::var("CARGO_FEATURE_ACCOUNTING").is_ok() { + builder = builder + .header("data/accounting.h") + .header("data/inspection.h"); + } for header in headers { builder = builder.clang_arg("-I").clang_arg(header.to_str().unwrap()); @@ -35,3 +66,123 @@ fn main() { let _ = file.write(s.as_bytes()); } + +fn fetch() -> io::Result<()> { + let output_base_path = output(); + let clone_dest_dir = format!("aom-{}", AOM_VERSION); + let _ = std::fs::remove_dir_all(output_base_path.join(&clone_dest_dir)); + let status = Command::new("git") + .current_dir(&output_base_path) + .arg("clone") + .arg("--depth=1") + .arg("-b") + .arg(format!("v{}", AOM_VERSION)) + .arg("https://aomedia.googlesource.com/aom") + .arg(&clone_dest_dir) + .status()?; + + if status.success() { + Ok(()) + } else { + Err(io::Error::new(io::ErrorKind::Other, "fetch failed")) + } +} + +fn build() -> io::Result<()> { + let source_dir = source(); + + let build_dir = "_build"; + let mut cmake = Command::new("cmake"); + cmake.current_dir(&source_dir); + cmake + .arg("-B") + .arg(build_dir) + .arg(format!( + "-DCMAKE_BUILD_TYPE={}", + if env::var("DEBUG").is_ok() { + "Debug" + } else { + "Release" + } + )) + .arg(format!( + "-DCMAKE_INSTALL_PREFIX={}", + search().to_string_lossy() + )) + .arg("-DCMAKE_INSTALL_LIBDIR=lib") + .arg("-DCONFIG_AV1_DECODER=1") + .arg("-DCONFIG_AV1_ENCODER=1") + .arg("-DBUILD_SHARED_LIBS=1") + .arg("-DENABLE_TESTS=0") + .arg("-DENABLE_EXAMPLES=0") + .arg("-DENABLE_DOCS=0"); + + if env::var("CARGO_FEATURE_ACCOUNTING").is_ok() { + // These features are needed for doing aomanalyzer-style analysis. + cmake + .arg("-DCONFIG_ACCOUNTING=1") + .arg("-DCONFIG_INSPECTION=1"); + } + + cmake.arg("./"); + + let output = cmake + .output() + .unwrap_or_else(|_| panic!("{:?} failed", cmake)); + if !output.status.success() { + println!("cmake: {}", String::from_utf8_lossy(&output.stdout)); + + return Err(io::Error::new( + io::ErrorKind::Other, + format!("cmake failed {}", String::from_utf8_lossy(&output.stderr)), + )); + } + + if !Command::new("make") + .current_dir(&source()) + .arg("-C") + .arg(build_dir) + .arg("-j") + .arg(available_parallelism().unwrap().to_string()) + .current_dir(&source()) + .status()? + .success() + { + return Err(io::Error::new(io::ErrorKind::Other, "make failed")); + } + + if !Command::new("make") + .current_dir(&source()) + .arg("-C") + .arg(build_dir) + .arg("install") + .status()? + .success() + { + return Err(io::Error::new(io::ErrorKind::Other, "make install failed")); + } + + Ok(()) +} + +fn output() -> PathBuf { + PathBuf::from(env::var("OUT_DIR").unwrap()) +} + +const AOM_VERSION: &str = "3.3.0"; + +fn source() -> PathBuf { + output().join(format!("aom-{}", AOM_VERSION)) +} + +fn search() -> PathBuf { + let mut absolute = env::current_dir().unwrap(); + absolute.push(&output()); + absolute.push("dist"); + + absolute +} + +fn pkg_config() -> PathBuf { + search().join("lib").join("pkgconfig") +} diff --git a/aom-sys/data/accounting.h b/aom-sys/data/accounting.h new file mode 100644 index 0000000..8ed2b05 --- /dev/null +++ b/aom-sys/data/accounting.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2016, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +#define AOM_ACCOUNTING_HASH_SIZE (1021) + +/* Max number of entries for symbol types in the dictionary (increase as + necessary). */ +#define MAX_SYMBOL_TYPES (256) + +/*The resolution of fractional-precision bit usage measurements, i.e., + 3 => 1/8th bits.*/ +#define AOM_ACCT_BITRES (3) + +typedef struct { + int16_t x; + int16_t y; +} AccountingSymbolContext; + +typedef struct { + AccountingSymbolContext context; + uint32_t id; + /** Number of bits in units of 1/8 bit. */ + uint32_t bits; + uint32_t samples; +} AccountingSymbol; + +/** Dictionary for translating strings into id. */ +typedef struct { + char *strs[MAX_SYMBOL_TYPES]; + int num_strs; +} AccountingDictionary; + +typedef struct { + /** All recorded symbols decoded. */ + AccountingSymbol *syms; + /** Number of syntax actually recorded. */ + int num_syms; + /** Raw symbol decoding calls for non-binary values. */ + int num_multi_syms; + /** Raw binary symbol decoding calls. */ + int num_binary_syms; + /** Dictionary for translating strings into id. */ + AccountingDictionary dictionary; +} AccountingSymbols; + +struct Accounting { + AccountingSymbols syms; + /** Size allocated for symbols (not all may be used). */ + int num_syms_allocated; + int16_t hash_dictionary[AOM_ACCOUNTING_HASH_SIZE]; + AccountingSymbolContext context; + uint32_t last_tell_frac; +}; + +void aom_accounting_init(Accounting *accounting); +void aom_accounting_reset(Accounting *accounting); +void aom_accounting_clear(Accounting *accounting); +void aom_accounting_set_context(Accounting *accounting, int16_t x, int16_t y); +int aom_accounting_dictionary_lookup(Accounting *accounting, const char *str); +void aom_accounting_record(Accounting *accounting, const char *str, + uint32_t bits); +void aom_accounting_dump(Accounting *accounting); +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus diff --git a/aom-sys/data/inspection.h b/aom-sys/data/inspection.h new file mode 100644 index 0000000..be3dd9b --- /dev/null +++ b/aom-sys/data/inspection.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +#ifndef AOM_AOM_AOMDX_H_ +typedef void (*aom_inspect_cb)(void *decoder, void *data); +#endif + +#define MAX_SEGMENTS 8 + +typedef struct insp_mv insp_mv; + +struct insp_mv { + int16_t row; + int16_t col; +}; + +typedef struct insp_mi_data insp_mi_data; + +struct insp_mi_data { + insp_mv mv[2]; + int16_t ref_frame[2]; + int16_t mode; + int16_t uv_mode; + int16_t bsize; + int16_t skip; + int16_t segment_id; + int16_t dual_filter_type; + int16_t filter[2]; + int16_t tx_type; + int16_t tx_size; + int16_t cdef_level; + int16_t cdef_strength; + int16_t cfl_alpha_idx; + int16_t cfl_alpha_sign; + int16_t current_qindex; + int16_t compound_type; + int16_t motion_mode; + int16_t intrabc; + int16_t palette; + int16_t uv_palette; +}; + +typedef struct insp_frame_data insp_frame_data; + +struct insp_frame_data { +#if CONFIG_ACCOUNTING + Accounting *accounting; +#endif + insp_mi_data *mi_grid; + int16_t frame_number; + int show_frame; + int frame_type; + int base_qindex; + int mi_rows; + int mi_cols; + int tile_mi_rows; + int tile_mi_cols; + int16_t y_dequant[MAX_SEGMENTS][2]; + int16_t u_dequant[MAX_SEGMENTS][2]; + int16_t v_dequant[MAX_SEGMENTS][2]; + // TODO(negge): add per frame CDEF data + int delta_q_present_flag; + int delta_q_res; + int show_existing_frame; +}; + +void ifd_init(insp_frame_data *fd, int frame_width, int frame_height); +void ifd_clear(insp_frame_data *fd); +int ifd_inspect(insp_frame_data *fd, void *decoder, int skip_not_transform); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus From d0a0f87e22a95f4ea5aa71cf7f388135b2678734 Mon Sep 17 00:00:00 2001 From: Josh Holmer Date: Sun, 19 Jun 2022 17:48:56 -0400 Subject: [PATCH 2/2] Test --- aom-sys/build.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/aom-sys/build.rs b/aom-sys/build.rs index 7c028b7..aa18aa9 100644 --- a/aom-sys/build.rs +++ b/aom-sys/build.rs @@ -109,9 +109,6 @@ fn build() -> io::Result<()> { "-DCMAKE_INSTALL_PREFIX={}", search().to_string_lossy() )) - .arg("-DCMAKE_INSTALL_LIBDIR=lib") - .arg("-DCONFIG_AV1_DECODER=1") - .arg("-DCONFIG_AV1_ENCODER=1") .arg("-DBUILD_SHARED_LIBS=1") .arg("-DENABLE_TESTS=0") .arg("-DENABLE_EXAMPLES=0")