Add docs, release tooling, reorganize pub exports

Author: Christian Kauhaus

.gitignore

@@ -1,4 +1,6 @@
-/target
+/dist/
 **/*.rs.bk
 .*.sw?
-tags
+/tags
+/target/
+/tmp/

Cargo.toml

@@ -5,6 +5,7 @@ authors = ["Christian Kauhaus <kc@flyingcircus.io>"]
 edition = "2018"
 description = "Rapid restore tool for backy"
 license = "BSD-3-Clause"
+repository = "https://github.com/ckauhaus/backy-extract"
 
 [dependencies]
 atty = "0.2"

Makefile

@@ -0,0 +1,28 @@
+# Set if your compiler cannot find liblzo2
+# LIBRARY_PATH = /path/to/liblzo2
+
+DESTDIR := dist
+
+all: release
+
+release: target/release/backy-extract
+
+target/release/backy-extract: src/*.rs src/*/*.rs
+	cargo build --release
+	strip $@
+
+VERSION = `cargo read-manifest | jq .version -r`
+PV = backy-extract-$(VERSION)
+
+dist: release
+	install -D target/release/backy-extract -t tmp/$(PV)/bin
+	install -D -m 0644 README.md -t tmp/$(PV)/share/doc
+	mkdir -p dist
+	tar czf dist/$(PV).tar.gz -C tmp $(PV)
+	rm -r tmp
+
+clean:
+	cargo clean
+	rm -rf tmp dist
+
+.PHONY: release dist clean

README.md

@@ -0,0 +1,55 @@
+backy-extract
+=============
+
+High-performance, standalone backup restore tool for
+[backy](https://bitbucket.org/flyingcircus/backy). backy-extract restores a
+specified backup revision to stdout or a file/block device target.
+
+
+Usage
+-----
+
+1. Pick a backup revision, for example using `backy status`.
+2. Create a restore target, for example with `lvm` or `rbd image`.
+3. Extract backup: `backy-extract /srv/backy/vm/Nym6uacWoXGb8VnbksM3yH /dev/rbd0`
+
+
+Interaction with backy
+----------------------
+
+`backy-extract` tries to acquire the *purge lock* before proceeding. So if it
+does not seem to start, check if there are running `backy` processes operating
+on the same backup.
+
+
+Sparse mode
+-----------
+
+Block devices are assumed to be zeroed (discarded) before restoring. If this is
+not the case, invoke `backy-extract` with `--sparse=never`.
+
+
+Compiling
+---------
+
+In general, `cargo build --release` should do the right thing. This application
+depends on [liblzo2](http://www.oberhumer.com/opensource/lzo/) being availabe.
+If you get compiler/linker errors, try compiling again with `export
+LIBRABY_PATH=/path/to/liblzo2`.
+
+A Makefile is supplied to create a statically linked release which should run on
+virtually every Linux x86_64 system.
+
+
+Hacking
+-------
+
+Please create issues and submit pull requests at
+https://github.com/ckauhaus/backy-extract/.
+
+
+Author
+------
+
+Contact [Christian Kauhaus](kc@flyingcircus.io) for questions, suggestions, and
+bug fixes.

src/chunkvec.rs

@@ -1,5 +1,5 @@
 use crate::backend::Backend;
-use crate::{pos2chunk, Chunk, ExtractError, CHUNKSZ_LOG};
+use crate::{pos2chunk, Chunk, Data, ExtractError, CHUNKSZ_LOG};
 
 use crossbeam::channel::Sender;
 use failure::{Fail, Fallible, ResultExt};
@@ -31,12 +31,6 @@ impl<'d> Revision<'d> {
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
-pub enum Data {
-    Some(Vec<u8>),
-    Zero,
-}
-
 type ChunkMap<'d> = BTreeMap<&'d str, SmallVec<[usize; 4]>>;
 
 /// All chunks of a revision, grouped by chunk ID.

src/lib.rs

@@ -1,16 +1,21 @@
+//! High-performance, multi-threaded backy image restore library.
+//!
+//! `backy_extract` reads an backup revision from a
+//! [backy](https://bitbucket.org/flyingcircus/backy) *chunked v2* data store, decompresses it on
+//! the fly and writes it to a restore target using pluggable writeout modules.
+
 mod backend;
 mod chunkvec;
 #[cfg(test)]
 mod test_helper;
 mod writeout;
 
 use self::backend::Backend;
-use self::chunkvec::{ChunkVec, Data};
+use self::chunkvec::{ChunkVec};
 pub use self::writeout::{RandomAccess, Stream};
-use self::writeout::{WriteOut, WriteOutBuilder};
 
 use console::{style, StyledObject};
-use crossbeam::channel::{bounded, unbounded, Receiver};
+use crossbeam::channel::{bounded, unbounded, Sender, Receiver};
 use crossbeam::thread;
 use failure::{Fail, Fallible, ResultExt};
 use fs2::FileExt;
@@ -19,18 +24,60 @@ use lazy_static::lazy_static;
 use memmap::MmapMut;
 use num_cpus;
 use smallvec::SmallVec;
+use std::fmt::Debug;
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use std::time::Instant;
 
-// Size of an uncompressed Chunk in the backy store.
+/// Size of an uncompressed Chunk in the backy store as 2's exponent.
 // This value must be a u32 because it is encoded as 32 bit uint the chunk file header.
 pub const CHUNKSZ_LOG: usize = 22; // 4 MiB
 
 lazy_static! {
     static ref ZERO_CHUNK: MmapMut = MmapMut::map_anon(1 << CHUNKSZ_LOG).expect("mmap");
 }
 
+/// WriteOut factory.
+///
+/// We use the factory approach to have only the minimum of parameters to the
+/// user-facing constructor. Further parameters from the Exctractor are supplied internally by
+/// invoking `build` to get the final WriteOut object.
+pub trait WriteOutBuilder {
+    type Impl: WriteOut + Sync + Send;
+    fn build(self, total_size: u64, threads: u8) -> Self::Impl;
+}
+
+/// Abstract writeout (restore) plugin.
+///
+/// A concrete writer is instantiated via `WriteOutBuilder.build()`.
+pub trait WriteOut: Debug {
+    /// Gets an unordered stream of `Chunk`s which must be written to the restore target according
+    /// to the chunks' sequence numbers. Writer must send the number of bytes written to the
+    /// `progress` channel to indicate restore progress in real time.
+    fn receive(self, chunks: Receiver<Chunk>, progress: Sender<usize>) -> Fallible<()>;
+
+    /// Short idenfication for user display. Should contain plugin type and file name.
+    fn name(&self) -> String;
+}
+
+/// Transport of a single image data chunk.
+///
+/// A chunk needs to be placed into all logical positions that are listed in the `seqs` attribute.
+/// Each seq starts at offset (seq << CHUNKSZ_LOG) bytes in the restored image.
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Chunk {
+    pub data: Data,
+    pub seqs: SmallVec<[usize; 4]>,
+}
+
+/// Block of uncompressed image contents of length (1 << CHUNKSZ_LOG).
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Data {
+    Some(Vec<u8>),
+    /// Shortcut if the whole block consists only of zeros.
+    Zero,
+}
+
 // Converts file position/size into chunk sequence number
 fn pos2chunk(pos: u64) -> usize {
     (pos >> CHUNKSZ_LOG) as usize
@@ -41,12 +88,6 @@ fn chunk2pos(seq: usize) -> u64 {
     (seq as u64) << CHUNKSZ_LOG
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
-pub struct Chunk {
-    data: Data,
-    seqs: SmallVec<[usize; 4]>,
-}
-
 fn purgelock(basedir: &Path) -> Fallible<File> {
     let f = File::create(basedir.join(".purge"))?;
     f.try_lock_exclusive()?;
@@ -94,10 +135,6 @@ impl Extractor {
         })
     }
 
-    fn default_threads() -> u8 {
-        (num_cpus::get() / 2).max(1).min(60) as u8
-    }
-
     /// Sets number of decompression threads. Heuristics apply in this method is never called.
     pub fn threads(&mut self, n: u8) -> &mut Self {
         if n > 0 {
@@ -106,6 +143,10 @@ impl Extractor {
         self
     }
 
+    fn default_threads() -> u8 {
+        (num_cpus::get() / 2).max(1).min(60) as u8
+    }
+
     /// Enables/disables a nice progress bar on stderr while restoring.
     pub fn progress(&mut self, show: bool) -> &mut Self {
         self.progress = if show {

src/writeout/mod.rs

@@ -4,24 +4,3 @@ mod stream;
 
 pub use self::randomaccess::RandomAccess;
 pub use self::stream::Stream;
-
-use super::Chunk;
-use crossbeam::channel::{Receiver, Sender};
-use failure::Fallible;
-use std::fmt::Debug;
-
-/// WriteOut factory.
-///
-/// We use the factory approach to have only the minimum of parameters to the
-/// user-facing constructor. Further parameters from the Exctractor are supplied internally by
-/// invoking `build` to get the final WriteOut object.
-pub trait WriteOutBuilder {
-    type Impl: WriteOut + Sync + Send;
-    fn build(self, total_size: u64, threads: u8) -> Self::Impl;
-}
-
-/// Abstracts over restore backends.
-pub trait WriteOut: Debug {
-    fn receive(self, chunks: Receiver<Chunk>, progress: Sender<usize>) -> Fallible<()>;
-    fn name(&self) -> String;
-}

src/writeout/randomaccess.rs

@@ -1,7 +1,5 @@
 use super::compat::write_all_at;
-use crate::{
-    chunk2pos, pos2chunk, Chunk, Data, PathExt, WriteOut, WriteOutBuilder, CHUNKSZ_LOG, ZERO_CHUNK,
-};
+use crate::{chunk2pos, pos2chunk, Chunk, Data, PathExt, WriteOut, WriteOutBuilder, CHUNKSZ_LOG, ZERO_CHUNK};
 
 use crossbeam::channel::{Receiver, Sender};
 use crossbeam::thread;
@@ -15,13 +13,22 @@ use std::io;
 use std::io::prelude::*;
 use std::path::{Path, PathBuf};
 
+/// File/block device restore target.
+///
+/// Chunks are written out-of-order as they are sent to the writer. Zeros are not written (i.e.,
+/// skipped over) if sparse mode is enabled.
 #[derive(Debug, Clone)]
 pub struct RandomAccess {
     path: PathBuf,
     sparse: Option<bool>,
 }
 
 impl RandomAccess {
+    /// Creates a builder which is finalized later by the [Extractor](struct.Extractor.html). If
+    /// the user explicitely requests sparse to be active or not, set `sparse` to some bool. If
+    /// sparse mode is not explicitely set, a heuristic is applied: files which can be truncated
+    /// are written sparsely. Block devices are written sparsely if a patrol read suggests that
+    /// they do not contain previously written data, e.g. after a discard.
     pub fn new<P: AsRef<Path>>(path: P, sparse: Option<bool>) -> Self {
         Self {
             path: path.as_ref().to_owned(),

src/writeout/stream.rs

@@ -1,5 +1,4 @@
-use super::{WriteOut, WriteOutBuilder};
-use crate::{Chunk, Data, CHUNKSZ_LOG, ZERO_CHUNK};
+use crate::{Chunk, Data, WriteOutBuilder, WriteOut, CHUNKSZ_LOG, ZERO_CHUNK};
 
 use crossbeam::channel::{Receiver, Sender};
 use failure::Fallible;
@@ -8,6 +7,10 @@ use std::fmt;
 use std::io::Write;
 use std::rc::Rc;
 
+/// Streaming restore target, i.e. write to stdout.
+///
+/// The incoming chunk stream is assembled into sequence order in memory. Chunks are
+/// written out eagerly to keep memory usage to a minimum.
 pub struct Stream<W: ?Sized + Write> {
     out: Box<W>,
 }