diff --git a/src/atomic64.rs b/src/atomic64.rs
index 6a20c031..66c19515 100644
--- a/src/atomic64.rs
+++ b/src/atomic64.rs
@@ -64,10 +64,10 @@ pub trait Atomic: Send + Sync {
     fn set(&self, val: Self::T);
     /// Get the value.
     fn get(&self) -> Self::T;
-    /// Increment the value by a given amount.
-    fn inc_by(&self, delta: Self::T);
-    /// Decrement the value by a given amount.
-    fn dec_by(&self, delta: Self::T);
+    /// Increment the value by a given amount returning the old value.
+    fn inc_by(&self, delta: Self::T) -> Self::T;
+    /// Decrement the value by a given amount returning the old value.
+    fn dec_by(&self, delta: Self::T) -> Self::T;
 }
 
 /// A atomic float.
@@ -106,7 +106,7 @@ impl Atomic for AtomicF64 {
     }
 
     #[inline]
-    fn inc_by(&self, delta: Self::T) {
+    fn inc_by(&self, delta: Self::T) -> Self::T {
         loop {
             let current = self.inner.load(Ordering::Acquire);
             let new = u64_to_f64(current) + delta;
@@ -117,14 +117,14 @@ impl Atomic for AtomicF64 {
                 Ordering::Relaxed,
             );
             if result.is_ok() {
-                return;
+                return u64_to_f64(current);
             }
         }
     }
 
     #[inline]
-    fn dec_by(&self, delta: Self::T) {
-        self.inc_by(-delta);
+    fn dec_by(&self, delta: Self::T) -> Self::T {
+        self.inc_by(-delta)
     }
 }
 
@@ -161,13 +161,13 @@ impl Atomic for AtomicI64 {
     }
 
     #[inline]
-    fn inc_by(&self, delta: Self::T) {
-        self.inner.fetch_add(delta, Ordering::Relaxed);
+    fn inc_by(&self, delta: Self::T) -> Self::T {
+        self.inner.fetch_add(delta, Ordering::Relaxed)
     }
 
     #[inline]
-    fn dec_by(&self, delta: Self::T) {
-        self.inner.fetch_sub(delta, Ordering::Relaxed);
+    fn dec_by(&self, delta: Self::T) -> Self::T {
+        self.inner.fetch_sub(delta, Ordering::Relaxed)
     }
 }
 
@@ -197,13 +197,13 @@ impl Atomic for AtomicU64 {
     }
 
     #[inline]
-    fn inc_by(&self, delta: Self::T) {
-        self.inc_by_with_ordering(delta, Ordering::Relaxed);
+    fn inc_by(&self, delta: Self::T) -> Self::T {
+        self.inc_by_with_ordering(delta, Ordering::Relaxed)
     }
 
     #[inline]
-    fn dec_by(&self, delta: Self::T) {
-        self.inner.fetch_sub(delta, Ordering::Relaxed);
+    fn dec_by(&self, delta: Self::T) -> Self::T {
+        self.inner.fetch_sub(delta, Ordering::Relaxed)
     }
 }
 
@@ -229,14 +229,19 @@ impl AtomicU64 {
     }
 
     /// Increment the value by a given amount with the provided memory ordering.
-    pub fn inc_by_with_ordering(&self, delta: u64, ordering: Ordering) {
-        self.inner.fetch_add(delta, ordering);
+    pub fn inc_by_with_ordering(&self, delta: u64, ordering: Ordering) -> u64 {
+        self.inner.fetch_add(delta, ordering)
     }
 
     /// Stores a value into the atomic integer, returning the previous value.
     pub fn swap(&self, val: u64, ordering: Ordering) -> u64 {
         self.inner.swap(val, ordering)
     }
+
+    /// Returns a reference to the inner atomic u64
+    pub fn inner(&self) -> &StdAtomicU64 {
+        &self.inner
+    }
 }
 
 #[cfg(test)]
diff --git a/src/gauge.rs b/src/gauge.rs
index 8e7f7028..759f07c6 100644
--- a/src/gauge.rs
+++ b/src/gauge.rs
@@ -72,15 +72,15 @@ impl<P: Atomic> GenericGauge<P> {
     /// Add the given value to the gauge. (The value can be
     /// negative, resulting in a decrement of the gauge.)
     #[inline]
-    pub fn add(&self, v: P::T) {
-        self.v.inc_by(v);
+    pub fn add(&self, v: P::T) -> P::T {
+        self.v.inc_by(v)
     }
 
     /// Subtract the given value from the gauge. (The value can be
     /// negative, resulting in an increment of the gauge.)
     #[inline]
-    pub fn sub(&self, v: P::T) {
-        self.v.dec_by(v);
+    pub fn sub(&self, v: P::T) -> P::T {
+        self.v.dec_by(v)
     }
 
     /// Return the gauge value.
@@ -288,4 +288,21 @@ mod tests {
         assert!(vec.remove_label_values(&[v1.clone()]).is_err());
         assert!(vec.remove_label_values(&[v1.clone(), v3.clone()]).is_err());
     }
+
+    #[test]
+    fn test_gauge_inc_sub_return_old() {
+        let opts = Opts::new("test", "test help")
+            .const_label("a", "1")
+            .const_label("b", "2");
+
+        let gauge = IntGauge::with_opts(opts).unwrap();
+        gauge.inc();
+        assert_eq!(gauge.get(), 1);
+        let val = gauge.add(20);
+        assert_eq!(val, 1);
+        assert_eq!(gauge.get(), 21);
+        let val = gauge.sub(10);
+        assert_eq!(val, 21);
+        assert_eq!(gauge.get(), 11);
+    }
 }
diff --git a/src/value.rs b/src/value.rs
index b9a9237e..6cc22eb6 100644
--- a/src/value.rs
+++ b/src/value.rs
@@ -65,8 +65,8 @@ impl<P: Atomic> Value<P> {
     }
 
     #[inline]
-    pub fn inc_by(&self, val: P::T) {
-        self.val.inc_by(val);
+    pub fn inc_by(&self, val: P::T) -> P::T {
+        self.val.inc_by(val)
     }
 
     #[inline]
@@ -80,7 +80,7 @@ impl<P: Atomic> Value<P> {
     }
 
     #[inline]
-    pub fn dec_by(&self, val: P::T) {
+    pub fn dec_by(&self, val: P::T) -> P::T {
         self.val.dec_by(val)
     }