introduce new function called scan_from_url

Author: Gal Ben David

Cargo.toml

@@ -1,14 +1,20 @@
 [package]
 name = "pyrepscan"
-version = "0.7.4"
+version = "0.8.0"
 authors = ["Gal Ben David <[email protected]>"]
 edition = "2018"
 description = "A Git Repository Secrets Scanner written in Rust"
 readme = "README.md"
 repository = "https://github.com/intsights/pyrepscan"
 homepage = "https://github.com/intsights/pyrepscan"
 license = "MIT"
-keywords = ["adblock", "ads", "adblocker", "rust", "brave", "abp", "pyo3"]
+keywords = [
+    "git",
+    "secrets",
+    "scanner",
+    "rust",
+    "pyo3",
+]
 
 [package.metadata.maturin]
 requires-python = ">=3.6"
@@ -33,13 +39,13 @@ regex = "1"
 rayon = "1.5"
 chrono = "0.4"
 parking_lot = "0.11"
+num_cpus = "1"
 
 [dependencies.git2]
 version = "0.13"
-default-features = false
 
 [dependencies.pyo3]
-version = "0.13.1"
+version = "0.13.2"
 features = ["extension-module"]
 
 [profile.release]

README.md

@@ -142,6 +142,22 @@ A sample result would look like this:
 ```
 
 
+```python
+def scan_from_url(
+    self,
+    url: str,
+    repository_path: str,
+    branch_glob_pattern: typing.Optional[str],
+    from_timestamp: typing.Optional[int],
+) -> typing.List[typing.Dict[str, str]]
+```
+The same as `scan` function but also clones a repository from a given URL into the provided repository path.
+- `url` - URL of a git repository.
+- `repository_path` - The path to clone the repository to
+- `branch_glob_pattern` - A glob pattern to filter branches for the scan. If None is sent, defaults to `*`.
+- `from_timestamp` - A UTC timestamp (Int) that only commits that were created after this timestamp would be included in the scan. If None is sent, defaults to `0`.
+
+
 ```python
 def get_file_content(
     self,

pyproject.toml

@@ -14,7 +14,7 @@ strip = true
 
 [tool.poetry]
 name = "pyrepscan"
-version = "0.7.4"
+version = "0.8.0"
 authors = ["Gal Ben David <[email protected]>"]
 description = "A Git Repository Secrets Scanner written in Rust"
 readme = "README.md"
@@ -49,3 +49,13 @@ gitpython = "*"
 wheel = "*"
 pytest-runner = "*"
 maturin = "*"
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = [
+    "--tb=native",
+    "--pythonwarnings=all",
+]
+testpaths = [
+    "tests",
+]

pyrepscan/pyrepscan.pyi

@@ -37,6 +37,14 @@ class GitRepositoryScanner:
         from_timestamp: typing.Optional[int],
     ) -> typing.List[typing.Dict[str, str]]: ...
 
+    def scan_from_url(
+        self,
+        url: str,
+        repository_path: str,
+        branch_glob_pattern: typing.Optional[str],
+        from_timestamp: typing.Optional[int],
+    ) -> typing.List[typing.Dict[str, str]]: ...
+
     def get_file_content(
         self,
         repository_path: str,

setup.cfg

@@ -2,12 +2,12 @@ use crate::rules_manager;
 
 use chrono::prelude::*;
 use git2::{Oid, Repository, Delta};
+use git2::Error;
 use parking_lot::Mutex;
 use rayon::prelude::*;
 use std::collections::HashMap;
-use std::sync::Arc;
-use git2::Error;
 use std::path::Path;
+use std::sync::Arc;
 
 fn scan_commit_oid(
     git_repo: &Repository,
@@ -152,13 +152,19 @@ pub fn scan_repository(
             }
         }
     }
+
+    let chunk_size = (oids.len() as f64 / (num_cpus::get() * 5) as f64).ceil() as usize;
     if !oids.is_empty() {
-        let chunk_size = (oids.len() as f64 / 100.0).ceil();
-        oids.par_chunks(chunk_size as usize).for_each_init(
-            || Repository::open(repository_path).unwrap(),
-            |git_repo, oids| {
+        oids.par_chunks(chunk_size).for_each(
+            |oids| {
+                let git_repo = Repository::open(repository_path).unwrap();
                 for oid in oids {
-                    scan_commit_oid(git_repo, oid, rules_manager, output_matches.clone()).unwrap_or(());
+                    scan_commit_oid(
+                        &git_repo,
+                        oid,
+                        rules_manager,
+                        output_matches.clone()
+                    ).unwrap_or(());
                 }
             },
         );

src/git_repository_scanner.rs

@@ -1,6 +1,7 @@
 mod git_repository_scanner;
 mod rules_manager;
 
+use git2::Repository;
 use parking_lot::Mutex;
 use pyo3::exceptions;
 use pyo3::prelude::*;
@@ -213,6 +214,52 @@ impl GitRepositoryScanner {
             Ok(matches.lock().to_object(py))
         }
     }
+
+    /// Scan a git repository for secrets. Rules shuld be loaded before calling this function.
+    ///
+    /// input:
+    ///     url: str -> URL of a git repository
+    ///     repository_path: str ->  The path to clone the repository to
+    ///     branch_glob_pattern: str ->  A blob pattern to match against the git branches names.
+    ///         Only matched branches will be scanned.
+    ///     from_timestamp: int = 0 ->  Unix epoch timestamp to start the scan from.
+    ///
+    /// returns:
+    ///     list[dict] -> List of matches
+    ///
+    /// example:
+    ///     grs.scan_from_url(
+    ///         url="https://github.com/rust-lang/git2-rs",
+    ///         repository_path="/path/to/repository",
+    ///         branch_glob_pattern="*",
+    ///     )
+    #[text_signature = "(url, repository_path, branch_glob_pattern, from_timestamp, /)"]
+    fn scan_from_url(
+        &self,
+        py: Python,
+        url: &str,
+        repository_path: &str,
+        branch_glob_pattern: Option<&str>,
+        from_timestamp: Option<i64>,
+    ) -> PyResult<Py<PyAny>> {
+        let matches = Arc::new(Mutex::new(Vec::<HashMap<&str, String>>::with_capacity(10000)));
+
+        if let Err(error) = Repository::clone(url, repository_path) {
+            return Err(exceptions::PyRuntimeError::new_err(error.to_string()));
+        };
+
+        if let Err(error) = git_repository_scanner::scan_repository(
+            repository_path,
+            branch_glob_pattern.unwrap_or("*"),
+            from_timestamp.unwrap_or(0),
+            &self.rules_manager,
+            matches.clone(),
+        ) {
+            Err(exceptions::PyRuntimeError::new_err(error.to_string()))
+        } else {
+            Ok(matches.lock().to_object(py))
+        }
+    }
 }
 
 /// PyRepScan is a Python library written in Rust. The library prodives an API to scan git repositories

src/lib.rs

@@ -13,6 +13,8 @@ def setUp(
         self,
     ):
         self.tmpdir = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmpdir.cleanup)
+
         bare_repo = git.Repo.init(
             path=self.tmpdir.name,
         )