cache file support range request

Author: timzaak

doc/Roadmap.md

@@ -24,11 +24,10 @@
 ### version 1.2.x
 - [x] more log for debug and trace
 - [x] basic CORS
-- [x] compress regression support(if client don't send accept-encoding header(including gzip), will send back data from file instead of cache)
+- [x] compress regression support(~~if client don't send accept-encoding header(including gzip), will send back data from file instead of cache~~ improved by v1.2.3)
 - [x] hot reload web static server(use SO_REUSEPORT *nix api, so it may be wrong with Windows).
 - [ ] ~~different config(cors/cache strategy/https and so on) for different domain.~~ (if this is needed?)
-
-### version 1.3.x
-- [ ] cache File `Range` Header support
-- [ ] drop self maintained `Warp`(copy out needed code from Warp)
-- [ ] `HEAD` request support or drop
+--- version 1.2.2
+- [x] cache File `Range` Header support
+- [ ] ~~drop self maintained `Warp`(copy out needed code from Warp)~~ (so much code from warp/fs.)
+- [x] `HEAD` request support or drop(support, don't need to do anything)

server/src/file_cache.rs

@@ -3,17 +3,14 @@ use anyhow::anyhow;
 use dashmap::DashMap;
 use flate2::read::GzEncoder;
 use flate2::Compression;
-use futures_util::sink::Buffer;
-use headers::{ETag, HeaderValue};
 use hyper::body::Bytes;
 use lazy_static::lazy_static;
 use mime::Mime;
 use std::collections::HashMap;
 use std::collections::HashSet;
 use std::fs::{File, Metadata};
-use std::io::{BufReader, Error, Read};
+use std::io::{BufReader, Read};
 use std::path::PathBuf;
-use std::str::FromStr;
 use std::sync::Arc;
 use std::time::Duration;
 use walkdir::WalkDir;
@@ -92,7 +89,7 @@ impl FileCache {
                                 .map_or("".to_string(), |x| x.to_string());
 
                             let data_block = if self.conf.max_size.unwrap_or(DEFAULT_MAX_SIZE)
-                                > metadata.len()
+                                < metadata.len()
                             {
                                 tracing::debug!("file block:{}", entry_path.display());
                                 DataBlock::FileBlock(ArcPath(Arc::new(entry_path)))

server/src/static_file_filter.rs

@@ -5,15 +5,17 @@ use headers::{
     AcceptRanges, CacheControl, ContentEncoding, ContentLength, ContentRange, ContentType,
     HeaderMapExt, LastModified, Range,
 };
+use hyper::body::Bytes;
 use hyper::Body;
 use percent_encoding::percent_decode_str;
+use std::ops::Bound;
 use std::path::Path;
 use std::sync::Arc;
 use tokio::fs::File;
 use tokio::io;
-use warp::fs::{bytes_range, conditionals, file_stream, optimal_buf_size, Cond, Conditionals};
+use warp::fs::{conditionals, file_stream, optimal_buf_size, Cond, Conditionals};
 use warp::host::Authority;
-use warp::http::{Response, StatusCode};
+use warp::http::{Method, Response, StatusCode};
 use warp::{reject, Filter, Rejection};
 
 //from warp::fs
@@ -31,6 +33,50 @@ fn sanitize_path(tail: &str) -> Result<String, Rejection> {
         Err(reject::not_found())
     }
 }
+#[derive(Debug)]
+pub struct BadRange;
+
+//from warp/fs
+fn bytes_range(range: Option<Range>, max_len: u64) -> Result<(u64, u64), BadRange> {
+    let range = if let Some(range) = range {
+        range
+    } else {
+        return Ok((0, max_len));
+    };
+
+    let ret = range
+        .iter()
+        .map(|(start, end)| {
+            let start = match start {
+                Bound::Unbounded => 0,
+                Bound::Included(s) => s,
+                Bound::Excluded(s) => s + 1,
+            };
+
+            let end = match end {
+                Bound::Unbounded => max_len,
+                Bound::Included(s) => {
+                    // For the special case where s == the file size
+                    if s == max_len {
+                        s
+                    } else {
+                        s + 1
+                    }
+                }
+                Bound::Excluded(s) => s,
+            };
+
+            if start < end && end <= max_len {
+                Ok((start, end))
+            } else {
+                tracing::trace!("unsatisfiable byte range: {}-{}/{}", start, end, max_len);
+                Err(BadRange)
+            }
+        })
+        .next()
+        .unwrap_or(Ok((0, max_len)));
+    ret
+}
 
 // copy from warp::fs file_reply
 async fn file_reply(
@@ -89,50 +135,83 @@ async fn file_reply(
     }
 }
 
+fn cache_reply(
+    item: &CacheItem,
+    bytes: &Bytes,
+    range: Option<Range>,
+    modified: Option<LastModified>,
+) -> Response<Body> {
+    let mut len = bytes.len() as u64;
+    // don't support multiple range
+    bytes_range(range, len)
+        .map(|(start, end)| {
+            let sub_len = end - start;
+            // range or all
+            let body = Body::from(bytes.slice((
+                Bound::Included(start as usize),
+                Bound::Excluded(end as usize),
+            )));
+            let mut resp = Response::new(body);
+
+            if sub_len != len {
+                *resp.status_mut() = StatusCode::PARTIAL_CONTENT;
+                resp.headers_mut().typed_insert(
+                    ContentRange::bytes(start..end, len).expect("valid ContentRange"),
+                );
+                len = sub_len;
+                resp.headers_mut().typed_insert(ContentLength(sub_len));
+            } else {
+                resp.headers_mut().typed_insert(ContentLength(len));
+            }
+            cache_item_to_response_header(&mut resp, &item, modified);
+            resp
+        })
+        .unwrap_or_else(|_| {
+            let mut resp = Response::new(Body::empty());
+            *resp.status_mut() = StatusCode::RANGE_NOT_SATISFIABLE;
+            resp.headers_mut()
+                .typed_insert(ContentRange::unsatisfied_bytes(len));
+            resp
+        })
+}
+
 async fn cache_or_file_reply(
     item: Arc<CacheItem>,
     conditionals: Conditionals,
     accept_encoding: Option<String>,
 ) -> Result<Response<Body>, Rejection> {
     let modified = item.meta.modified().map(LastModified::from).ok();
-
-    let resp = match conditionals.check(modified) {
+    match conditionals.check(modified) {
         Cond::NoBody(resp) => Ok(resp),
         Cond::WithBody(range) => match &item.data {
             DataBlock::CacheBlock {
                 bytes,
-                compressed,
                 path,
+                compressed,
             } => {
-                if (accept_encoding
+                let compressed = *compressed;
+                let client_accept_gzip = accept_encoding
                     .as_ref()
                     .filter(|x| x.contains("gzip"))
-                    .is_none())
-                    && *compressed
-                {
-                    tracing::debug!(
-                        "{} don't use cache for accept_encoding:{:?}",
-                        path.as_ref().display(),
-                        &accept_encoding
-                    );
+                    .is_some();
+                //gzip header, compressed_data
+                //true,true => cache
+                //true, false => cache without content-encoding
+                //false,false => cache without content-encoding
+                //false, true => file
+                if !client_accept_gzip && compressed {
                     file_reply(&item, path.as_ref(), range, modified).await
                 } else {
-                    let mut resp = Response::new(Body::from(bytes.clone()));
-                    cache_item_to_response_header(&mut resp, &item, modified);
-                    resp.headers_mut()
-                        .typed_insert(ContentLength(bytes.len() as u64));
-                    if *compressed {
-                        resp.headers_mut()
-                            .typed_insert(ContentLength(bytes.len() as u64));
+                    let mut resp = cache_reply(item.as_ref(), bytes, range, modified);
+                    if client_accept_gzip && compressed {
                         resp.headers_mut().typed_insert(ContentEncoding::gzip());
                     }
                     Ok(resp)
                 }
             }
             DataBlock::FileBlock(path) => file_reply(&item, path.as_ref(), range, modified).await,
         },
-    };
-    resp
+    }
 }
 
 fn cache_item_to_response_header(