Merge pull request #24 from oiwn/dev

Updating to new minor version

Author: oiwn

.github/workflows/ci.yml

@@ -103,26 +103,6 @@ jobs:
         with:
           command: "test"
 
-  fmt:
-    name: "Cargo format"
-    runs-on: "ubuntu-latest"
-    steps:
-      - name: "Check out the repo"
-        uses: actions/checkout@v3
-
-      - uses: "actions-rs/toolchain@v1"
-        with:
-          profile: "minimal"
-          toolchain: "stable"
-          override: true
-
-      - run: "rustup component add rustfmt"
-
-      - uses: "actions-rs/cargo@v1"
-        with:
-          command: "fmt"
-          args: "--all -- --check"
-
   clippy:
     name: "Cargo clippy"
     runs-on: "ubuntu-latest"

.github/workflows/coverage.yml

@@ -1 +1,45 @@
+name: Coverage
 
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  coverage:
+    name: Code coverage
+    runs-on: ubuntu-latest
+    
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: llvm-tools-preview
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+        
+      - name: Generate code coverage
+        run: cargo llvm-cov --workspace --lcov --output-path lcov.info
+        
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: lcov.info
+          fail_ci_if_error: true
+          verbose: true

.github/workflows/publish.yml

@@ -1,31 +1,59 @@
-name: Publish on crates.io
+name: Publish to crates.io
 
 on:
-  pull_request:
-    types:
-      - closed
+  push:
     branches:
       - main
+    paths:
+      - 'Cargo.toml'
 
 jobs:
-  publish:
-    if: github.event.pull_request.merged == true
+  check-version:
     runs-on: ubuntu-latest
+    outputs:
+      should_publish: ${{ steps.compare_versions.outputs.should_publish }}
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
-        fetch-depth: 0
-        ref: main
+        fetch-depth: 0  # Fetch all history for comparing versions
+
+    - name: Compare versions
+      id: compare_versions
+      run: |
+        # Get the current version from Cargo.toml
+        CURRENT_VERSION=$(grep '^version = ' Cargo.toml | cut -d '"' -f 2)
+        echo "Current version: $CURRENT_VERSION"
+
+        # Get the previous version from the last commit
+        git checkout HEAD~1
+        PREVIOUS_VERSION=$(grep '^version = ' Cargo.toml | cut -d '"' -f 2)
+        echo "Previous version: $PREVIOUS_VERSION"
+
+        # Compare versions
+        if [ "$CURRENT_VERSION" != "$PREVIOUS_VERSION" ]; then
+          echo "Version changed from $PREVIOUS_VERSION to $CURRENT_VERSION"
+          echo "should_publish=true" >> "$GITHUB_OUTPUT"
+        else
+          echo "Version unchanged"
+          echo "should_publish=false" >> "$GITHUB_OUTPUT"
+        fi
+
+  publish:
+    needs: check-version
+    if: needs.check-version.outputs.should_publish == 'true'
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
 
     - name: Setup Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        profile: minimal
-        toolchain: stable
-        override: true
-     
+      uses: dtolnay/rust-toolchain@stable
+
+    - name: Check if package is publishable
+      run: cargo publish --dry-run
+
     - name: Publish to crates.io
-      if: steps.check_version.outputs.version_updated == 'true'
       run: cargo publish --token ${{ secrets.CRATES_IO_TOKEN }}
+
+

.gitignore

@@ -9,3 +9,5 @@
 /data
 *.profraw
 all_code.txt
+.code
+.amc.toml

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "dom-content-extraction"
-version = "0.3.6"
+version = "0.3.7"
 
 description = "Rust implementation of Content extraction via text density paper"
 license = "MPL-2.0"

notes.org

@@ -1,14 +1,17 @@
 #+title: Notes
 
+* Unicode documents handling
+** TODO I think there are problems processing unicode.
+
 * Microtasks
-** Better split for CI/CD workflows
+** DONE Better split for CI/CD workflows
 ** DONE 13 Nov - updating, removing unwraps.
 *** DONE add custom errors, remove unwraps
 *** DONE forbid unwrap in linting rules
-** TODO coverage should be >80%
-** TODO integrate cargo-tarpaulin or gcov into the github ci pipeline
+** DONE coverage should be >80%
+** DONE integrate cargo-tarpaulin or gcov into the github ci pipeline
 ** TODO cargo publish workflow
-** TODO add clear copy-pastable example into readme.md
+** DONE add clear copy-pastable example into readme.md
 ** DONE need "examples" command for "lorem ipsum" test page
 ** DONE need "benchmark"
 ** DONE need to implement testing among real data set

src/lib.rs

@@ -885,4 +885,35 @@ mod tests {
 
         assert!(!extracted_content.contains("Menu"));
     }
+
+    #[test]
+    fn test_document_node_handling() {
+        // Create a minimal HTML document that forces Document node traversal
+        let html = r#"<!DOCTYPE html><html><body><div>Test</div></body></html>"#;
+        let document = Html::parse_document(html);
+
+        // Get the root node which should be a Document node
+        let root_node = document.tree.root();
+        assert!(matches!(root_node.value(), scraper::Node::Document));
+
+        // Create a DensityTree starting from root to ensure Document node is encountered
+        let mut density_tree = DensityTree::new(root_node.id());
+        DensityTree::build_density_tree(
+            root_node,
+            &mut density_tree.tree.root_mut(),
+            1,
+        );
+
+        // If we reach here without panicking and the tree is built,
+        // it means the Document node was properly skipped
+        assert!(density_tree.tree.root().children().count() > 0);
+
+        // Verify the content is still processed despite skipping Document node
+        let text_nodes: Vec<_> = density_tree
+            .tree
+            .nodes()
+            .filter(|n| n.value().char_count > 0)
+            .collect();
+        assert!(!text_nodes.is_empty());
+    }
 }