Allow for subdirectories, add dom cleaning

Author: mkardas

Makefile

@@ -1,33 +1,41 @@
 DATA_DIR = ../data
 ANNOTATIONS_DIR = $(DATA_DIR)/annotations
-ARCHIVES_DIR = $(DATA_DIR)/arxiv/sources
-UNPACKED_DIR = $(DATA_DIR)/arxiv/unpacked_sources
-HTMLS_DIR = $(DATA_DIR)/arxiv/htmls
-TABLES_DIR = $(DATA_DIR)/arxiv/tables
+ARXIV_DIR = $(DATA_DIR)/arxiv
+ARCHIVES_DIR = $(ARXIV_DIR)/sources
+UNPACKED_DIR = $(ARXIV_DIR)/unpacked_sources
+HTMLS_DIR = $(ARXIV_DIR)/htmls
+FIXED_HTMLS_DIR = $(ARXIV_DIR)/htmls-clean
+TABLES_DIR = $(ARXIV_DIR)/tables
 
-ARCHIVES = $(wildcard $(ARCHIVES_DIR)/*)
-UNPACKS  = $(patsubst $(ARCHIVES_DIR)/%,$(UNPACKED_DIR)/%,$(ARCHIVES))
-HTMLS    = $(patsubst $(ARCHIVES_DIR)/%,$(HTMLS_DIR)/%.html,$(ARCHIVES))
-TABLES   = $(patsubst $(ARCHIVES_DIR)/%,$(TABLES_DIR)/%,$(ARCHIVES))
+ARCHIVES    = $(wildcard $(ARCHIVES_DIR)/**/*.gz)
+UNPACKS     = $(patsubst $(ARCHIVES_DIR)/%.gz,$(UNPACKED_DIR)/%,$(ARCHIVES))
+HTMLS       = $(patsubst $(ARCHIVES_DIR)/%.gz,$(HTMLS_DIR)/%.html,$(ARCHIVES))
+FIXED_HTMLS = $(patsubst $(ARCHIVES_DIR)/%.gz,$(FIXED_HTMLS_DIR)/%.html,$(ARCHIVES))
+TABLES      = $(patsubst $(ARCHIVES_DIR)/%.gz,$(TABLES_DIR)/%,$(ARCHIVES))
 
-$(shell mkdir -p "$(DATA_DIR)" "$(HTMLS_DIR)" "$(TABLES_DIR)")
+$(shell mkdir -p "$(DATA_DIR)" "$(UNPACKED_DIR)" "$(HTMLS_DIR)" "$(FIXED_HTMLS_DIR)" "$(TABLES_DIR)")
 
 .PHONY: all
 all:	$(ANNOTATIONS_DIR)/pdfs-urls.csv $(ANNOTATIONS_DIR)/sources-urls.csv extract_all
 
 extract_all: $(TABLES)
 
+fix_htmls_all: $(FIXED_HTMLS)
+
 convert_all: $(HTMLS)
 
-$(TABLES): $(TABLES_DIR)/%: $(HTMLS_DIR)/%.html
-	python ./extract_tables.py $^ --outdir $(TABLES_DIR)
+$(TABLES): $(TABLES_DIR)/%: $(FIXED_HTMLS_DIR)/%.html
+	python ./extract_tables.py $^ --outdir $@
+
+$(FIXED_HTMLS): $(FIXED_HTMLS_DIR)/%: $(HTMLS_DIR)/%
+	./clean_html.sh $^ $@
 
 $(HTMLS): $(HTMLS_DIR)/%.html: $(UNPACKED_DIR)/%
-	./docker-latex2html.sh $(HTMLS_DIR) $^
+	./docker-latex2html.sh $^ $@
 
 unpack_all: $(UNPACKS)
 
-$(UNPACKS): $(UNPACKED_DIR)/%: $(ARCHIVES_DIR)/%
+$(UNPACKS): $(UNPACKED_DIR)/%: $(ARCHIVES_DIR)/%.gz
 	./unpack-sources.sh $^ $@
 
 $(ANNOTATIONS_DIR)/pdfs-urls.csv:	$(ANNOTATIONS_DIR)/papers-urls.csv

clean_html.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+SOURCE=$(realpath "$1")
+[ ! -f "$SOURCE" ] && echo "File $SOURCE not found." >&2 && exit 1
+mkdir -p $(dirname "$2")
+OUTPUT=$(realpath "$2")
+
+docker run --rm -v "$SOURCE":/files/index.html:ro --entrypoint '' zenika/alpine-chrome:73 chromium-browser --headless --disable-gpu --disable-software-rasterizer --no-sandbox --dump-dom /files/index.html > "$OUTPUT"

docker-latex2html.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
-OUTPUT_DIR=$(realpath "$1")
-ARCHIVE=$(realpath "$2")
-FILENAME=$(basename "$ARCHIVE")
+SOURCE_DIR=$(realpath "$1") 	#~/arxiv/unpacked/1701/1701.xyz
+[ ! -d "$SOURCE_DIR" ] && echo "Directory $SOURCE_DIR not found." >&2 && exit 1
+mkdir -p $(dirname "$2")
+OUTPUT=$(realpath "$2")		#~/arxiv/htmls/1701/1701.xyz.html
+OUTPUT_DIR=$(dirname "$OUTPUT")	#~/arxiv/htmls/1701
+FILENAME=$(basename "$OUTPUT") #1701.xyz.html
 
-docker run --rm --stop-timeout 60 -v $PWD/latex2html.sh:/files/latex2html.sh:ro -v "$ARCHIVE":/files/ro-source:ro -v "$OUTPUT_DIR":/files/htmls niccokunzmann/ci-latex /files/latex2html.sh "$FILENAME"
+docker run --rm -v $PWD/latex2html.sh:/files/latex2html.sh:ro -v "$SOURCE_DIR":/files/ro-source:ro -v "$OUTPUT_DIR":/files/htmls niccokunzmann/ci-latex /files/latex2html.sh "$FILENAME"

extract_tables.py

@@ -114,7 +114,7 @@ def set_ids_by_labels(soup):
 def extract_tables(filename, outdir):
     with open(filename, "rb") as f:
         html = f.read()
-    outdir = Path(outdir) / Path(filename).stem
+    outdir = Path(outdir)
     outdir.mkdir(parents=True, exist_ok=True)
     soup = BeautifulSoup(html, "lxml")
     flatten_tables(soup)

latex2html.sh

@@ -9,8 +9,8 @@ cp -r "$RO_SOURCE_DIR" "$SOURCE_DIR"
 cd "$SOURCE_DIR"
 MAINTEX=$(find . -type f -iname "*.tex" -print0 | xargs -0 grep -l documentclass | head -1)
 echo $MAINTEX
-timeout -s KILL 30 htlatex "$MAINTEX" '' '' '' '-interaction=nonstopmode'
+timeout -s KILL 60 htlatex "$MAINTEX" '' '' '' '-interaction=nonstopmode'
 
 FILENAME=$(basename $MAINTEX)
 FILENAME="${FILENAME%.tex}.html"
-cp "$SOURCE_DIR/$FILENAME" "$OUTPUT_DIR/$OUTNAME.html"
+cp "$SOURCE_DIR/$FILENAME" "$OUTPUT_DIR/$OUTNAME"