Skip to content

Commit 735ceb2

Browse files
committed
fix remove_artifacts/headers/footers for "page <num>"
When a header/footer span has text with just a digit changing then it's a page number element, which should be completely removed (not needing the first occurence exception)
1 parent 9cf0bc8 commit 735ceb2

1 file changed

Lines changed: 25 additions & 18 deletions

File tree

src/document.rs

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -473,7 +473,7 @@ pub struct PdfDocument {
473473
/// first appearance is often the document's cover-page title that just
474474
/// happens to echo into the header band on every page (B3: pdfa_010
475475
/// would otherwise drop "University of Oklahoma 2009").
476-
running_artifact_signatures: Mutex<Option<std::collections::HashMap<String, usize>>>,
476+
running_artifact_signatures: Mutex<Option<std::collections::HashMap<String, (usize, bool)>>>,
477477
/// Memoised result of [`PdfDocument::output_intent_cmyk_profile`].
478478
///
479479
/// The accessor walks `/OutputIntents` and decodes + parses the ICC
@@ -14791,7 +14791,7 @@ impl PdfDocument {
1479114791
/// the page, and keeps entries that recur on >=50% of pages.
1479214792
fn ensure_running_artifact_signatures(
1479314793
&self,
14794-
) -> Result<std::collections::HashMap<String, usize>> {
14794+
) -> Result<std::collections::HashMap<String, (usize, bool)>> {
1479514795
{
1479614796
let guard = self.running_artifact_signatures.lock_or_recover();
1479714797
if let Some(ref map) = *guard {
@@ -14894,38 +14894,38 @@ impl PdfDocument {
1489414894
}
1489514895
}
1489614896
let threshold = (page_count as f32 * 0.5).ceil() as usize;
14897-
let signatures: std::collections::HashMap<String, usize> = occurrences
14897+
let signatures: std::collections::HashMap<String, (usize, bool)> = occurrences
1489814898
.into_iter()
14899-
.filter(|(sig, (count, _))| {
14900-
let variants = literal_variants.get(sig).map(|s| s.len()).unwrap_or(0);
14899+
.filter_map(|(sig, (count, _))| {
14900+
let variants = literal_variants.get(&sig).map(|s| s.len()).unwrap_or(0);
1490114901
// Varying-literal path (page numbers / dates): the digits change per
1490214902
// page. Recurs on >=50% of body pages.
14903-
if *count >= threshold.max(2) && variants >= 2 {
14904-
return true;
14903+
if count >= threshold.max(2) && variants >= 2 {
14904+
return Some((sig, true));
1490514905
}
1490614906
// Item 6B (M5): CONSTANT-literal pagination/citation (DOI, volume/
1490714907
// article, journal URL + digit). The literal never changes, so the
1490814908
// varying-literal gate above misses it. Require a STRICTER >=60%
1490914909
// recurrence AND the narrow citation/URL shape gate, so substantive
1491014910
// repeated content (facility names, titles) is never suppressed.
1491114911
let strict = (page_count as f32 * 0.6).ceil() as usize;
14912-
if *count >= strict.max(2)
14912+
if count >= strict.max(2)
1491314913
&& variants < 2
1491414914
&& literal_variants
14915-
.get(sig)
14915+
.get(&sig)
1491614916
.and_then(|s| s.iter().next())
1491714917
.is_some_and(|lit| Self::looks_like_stable_pagination(lit))
1491814918
{
14919-
return true;
14919+
return Some((sig, false));
1492014920
}
14921-
false
14921+
None
1492214922
})
14923-
.map(|(sig, _)| {
14923+
.map(|(sig, is_varying)| {
1492414924
// Use the earliest page the signature appeared on — which
1492514925
// may be a body-content-skipped cover page that `occurrences`
1492614926
// didn't count toward the threshold but `first_seen_any` did.
1492714927
let first = first_seen_any.get(&sig).copied().unwrap_or(0);
14928-
(sig, first)
14928+
(sig, (first, is_varying))
1492914929
})
1493014930
.collect();
1493114931
*self.running_artifact_signatures.lock_or_recover() = Some(signatures.clone());
@@ -15009,13 +15009,20 @@ impl PdfDocument {
1500915009
continue;
1501015010
}
1501115011
let sig = Self::normalize_artifact_signature(trimmed);
15012-
if let Some(&first_seen_on) = signatures.get(&sig) {
15013-
// Keep the first appearance — it's usually the document
15014-
// cover-page title that got classified as chrome only
15015-
// because later pages repeat it as a running header (B3).
15016-
if page_index == first_seen_on {
15012+
if let Some(&(first_seen_on, is_varying)) = signatures.get(&sig) {
15013+
// is_varying=false: constant-literal string that already passed
15014+
// the strict looks_like_stable_pagination gate (DOI/volume/
15015+
// journal-URL citations) — never an arbitrary repeated title.
15016+
// Skip the first occurrence: it may be a title page
15017+
// that later pages happen to echo as a running citation (B3),
15018+
// so only strip it from second occurrence onward
15019+
if page_index == first_seen_on && !is_varying {
1501715020
continue;
1501815021
}
15022+
// is_varying=true: page numbers/dates, where the literal text
15023+
// changes per page. Such a signature can't be a one-off
15024+
// heading by definition, so every occurrence — including the
15025+
// first — is real pagination chrome to remove.
1501915026
s.artifact_type = Some(crate::extractors::text::ArtifactType::Pagination(
1502015027
crate::extractors::text::PaginationSubtype::Other,
1502115028
));

0 commit comments

Comments
 (0)