update-development-artifacts.py

#!/usr/bin/env python3
import os, os.path
import tempfile
import shutil
import subprocess
import filecmp
import pathlib
import sys
import datetime


##############################
#
#   Functions
#
##############################

import os

def normalize_newlines_to_usfm(directory_path):
    """
    Normalizes newlines in all files within the specified directory by replacing double newlines (\n\n) 
    with a single newline (\n). The files are overwritten with the updated content.

    Args:
        directory_path (str): Path to the directory containing the files to process.
    """
    if not os.path.isdir(directory_path):
        print(f"Error: The provided path '{directory_path}' is not a directory.")
        return

    for filename in os.listdir(directory_path):
        file_path = os.path.join(directory_path, filename)
        
        # Skip directories; only process files
        if not os.path.isfile(file_path):
            continue

        try:
            # Read the file content
            with open(file_path, 'r', encoding='utf-8') as file:
                content = file.read()

            # Replace all \n\n with \n
            while '\n\n' in content:
                content = content.replace('\n\n', '\n') 

            # Write the updated content back to the file
            with open(file_path, 'w', encoding='utf-8') as file:
                file.write(content)
            
        except Exception as e:
            print(f"Failed to process file: {file_path}. Error: {e}")


def lastModified(book):
    d = 'cd ' + sourceDir + ' ; git log -1 --format="%ad" -- "' + book.sourceFileName() + '"'
    d = subprocess.getoutput(d)
    d = datetime.datetime.strptime(d, '%a %b %d %X %Y %z')
    n = str(d.year) + '.' + str(d.month) + '.' + str(d.day)
    return n

def updateUSFM():
    print('******* UPDATING ' + tags + ' *******')
    print('  Building from: ' + sourceDir)
    print('          using: ' + toolsDir)
    print('           usfm: ' + usfmDir)
    print('            rtf: ' + rtfDir)
    print('            tmp: ' + tempDir)

    run = toolsDir + '/usfm-tools variant -s ' + sourceDir + ' -d ' + tempDir + ' -t ' + tags + ' -b ' + booklist + ' ' + swap
    subprocess.run(run, shell=True)

    #
    # Remove double \n\n (which are nice but not valid USFM)
    #
    normalize_newlines_to_usfm(tempDir)

    isDifferent = False
    for book in books:
        fm = tempDir  + '/' + book.fileName()
        to = usfmDir + '/' + book.fileName()

        if os.path.isfile(fm) and \
           (not os.path.isfile(to) or not filecmp.cmp(fm, to)):

            # We are different from last run
            isDifferent = True

            # Update USFM
            shutil.move(fm, to)

            # Update RTF
            subprocess.run('rm ' + tempDir2 + '/*', shell=True)
            shutil.copy(to, tempDir2)
            run = toolsDir + '/usfm-tools transform --target=rtf --usfmDir=' + tempDir2 + ' --builtDir=' + rtfDir + ' --config=' + config + ' --name="' + buildId + '-' + book.name() + '"'
            subprocess.run(run, shell=True)

            # Update MD
            subprocess.run('rm ' + tempDir2 + '/*', shell=True)
            shutil.copy(to, tempDir2)
            run = toolsDir + '/usfm-tools transform --target=md --usfmDir=' + tempDir2 + ' --builtDir=' + mdDir + ' --config=' + config + ' --name="' + buildId + '-' + book.name() + '"'
            subprocess.run(run, shell=True)

            print('Updated ' + book.fileName())

    run = toolsDir + '/usfm-tools check --oeb -s ' + usfmDir
    subprocess.run(run, shell=True)

    # if isDifferent:
    #         # Update Accordance
    #         subprocess.run('rm ' + tempDir2 + '/*', shell=True)
    #         shutil.copy(to, tempDir2)
    #         run = toolsDir + '/usfm-tools transform --target=accordance --usfmDir=' + usfmDir + ' --builtDir=' + indexDir + ' --config=' + config + ' --name="' + buildId + '"'
    #         subprocess.run(run, shell=True)
    #         print('Updated Accordance')

    print('******* UPDATING INDEX *******')
    f = open (indexDir + '/table.html', 'w')
    f.write(table(books))
    f.close()

def templateForBook(b):
    template = """
        <td>
          <span class="{}">{}</span><br>
          <span class="comment"><a href="https://github.com/openenglishbible/Open-English-Bible/blob/master/artifacts/us/rtf/oeb-working-{}.rtf">rtf</a> <a href="https://github.com/openenglishbible/Open-English-Bible/blob/master/source/{}.usfm.db">usfm.db</a><br>{}
          </span>
        </td>
    """
    status = 'absent'
    try:
        f = open(sourceDir + '/' + b.sourceFileName())
        c = f.read()
        f.close()
        if c.find(r'\rem IN RELEASE') > 0: status = 'released'
        elif c.find(r'\rem OK FOR RELEASE') > 0: status = 'checked'
        elif c.find(r'\rem DEVELOPMENT ONLY') > 0: status = 'unchecked'
        else: status = 'unchecked'
        d = lastModified(b)
    except Exception as e:
        status = 'absent'
        d = ''
    return template.format(status, b.full, b.name(), b.name(), d)

def headerLine(s):
    return '<tr class="header"><td colspan="5">' + s + '</td></tr>'

def section(name, books):
    table = headerLine(name)
    if len(books) <= 5:
        table = table + subsection(books)
    elif len(books) <= 10:
        table = table + subsection(books[:5])
        table = table + subsection(books[5:])
    elif len(books) <= 15:
        table = table + subsection(books[:5])
        table = table + subsection(books[5:10])
        table = table + subsection(books[10:])
    elif len(books) <= 20:
        table = table + subsection(books[:5])
        table = table + subsection(books[5:10])
        table = table + subsection(books[10:15])
        table = table + subsection(books[15:20])
    elif len(books) <= 25:
        table = table + subsection(books[:5])
        table = table + subsection(books[5:10])
        table = table + subsection(books[10:15])
        table = table + subsection(books[15:20])
        table = table + subsection(books[20:25])
    return table

def subsection(books):
    table = '<tr>'
    for b in books:
        table = table + templateForBook(b)
    return table + '</tr>'

def table(books):
    table = """<table style="border-spacing: 10px;">"""
    table = table + section('Pentateuch', books[1:6])  # Remember, 'first book' is preface
    table = table + section('Histories', books[6:18])
    table = table + section('Wisdom', books[18:23])
    table = table + section('Major Prophets', books[23:28])
    table = table + section('Minor Prophets', books[28:40])
    table = table + section('Gospels and Acts', books[40:45])
    table = table + section('Letters and Revelation', books[45:67])
    table = table + '</table>'   
    return table

##############################
#
#   Books Data
#
##############################

class Book(object):
  def __init__(self, number, sil, full):
    self.number = number
    self.sil = sil
    self.full = full
    
  def sourceFileName(self):
    return self.name() + '.usfm.db'

  def fileName(self):
    return self.name() + '.usfm'

  def name(self):
    return self.number + '-' + self.full
    
books = []

books.append( Book('00', 'FRT', 'Front Page'      ))
books.append( Book('01', 'GEN', 'Genesis'         ))
books.append( Book('02', 'EXO', 'Exodus'          ))
books.append( Book('03', 'LEV', 'Leviticus'       ))
books.append( Book('04', 'NUM', 'Numbers'         ))
books.append( Book('05', 'DEU', 'Deuteronomy'     ))

books.append( Book('06', 'JOS', 'Joshua'          ))
books.append( Book('07', 'JUD', 'Judges'          ))
books.append( Book('08', 'RUT', 'Ruth'            ))

books.append( Book('09', '1SA', '1 Samuel'        ))
books.append( Book('10', '2SA', '2 Samuel'        ))
books.append( Book('11', '1KI', '1 Kings'         ))
books.append( Book('12', '2KI', '2 Kings'         ))
books.append( Book('13', '1CH', '1 Chronicles'    ))
books.append( Book('14', '2CH', '2 Chronicles'    ))
books.append( Book('15', 'EZR', 'Ezra'            ))
books.append( Book('16', 'NEH', 'Nehemiah'        ))
books.append( Book('17', 'EST', 'Esther'          ))
books.append( Book('18', 'JOB', 'Job'             ))

books.append( Book('19', 'PSA', 'Psalms'          ))
books.append( Book('20', 'PRO', 'Proverbs'        ))
books.append( Book('21', 'ECC', 'Ecclesiastes'    ))
books.append( Book('22', 'SNG', 'Song of Songs'   ))

books.append( Book('23', 'ISA', 'Isaiah'          ))
books.append( Book('24', 'JER', 'Jeremiah'        ))
books.append( Book('25', 'LAM', 'Lamentations'    ))
books.append( Book('26', 'EZK', 'Ezekiel'         ))
books.append( Book('27', 'DAN', 'Daniel'          ))

books.append( Book('28', 'HOS', 'Hosea'           ))
books.append( Book('29', 'JOL', 'Joel'            ))
books.append( Book('30', 'AMO', 'Amos'            ))
books.append( Book('31', 'OBA', 'Obadiah'         ))
books.append( Book('32', 'JON', 'Jonah'           ))
books.append( Book('33', 'MIC', 'Micah'           ))
books.append( Book('34', 'NAM', 'Nahum'           ))
books.append( Book('35', 'HAB', 'Habakkuk'        ))
books.append( Book('36', 'ZEP', 'Zephaniah'       ))
books.append( Book('37', 'HAG', 'Haggai'          ))
books.append( Book('38', 'ZEC', 'Zechariah'       ))
books.append( Book('39', 'MAL', 'Malachi'         ))

books.append( Book('40', 'MAT', 'Matthew'         ))
books.append( Book('41', 'MRK', 'Mark'            ))
books.append( Book('42', 'LUK', 'Luke'            ))
books.append( Book('43', 'JON', 'John'            ))
books.append( Book('44', 'ACT', 'Acts'            ))

books.append( Book('45', 'ROM', 'Romans'))
books.append( Book('46', '1CO', '1 Corinthians'))
books.append( Book('47', '2CO', '2 Corinthians'))
books.append( Book('48', 'GAL', 'Galatians'))
books.append( Book('49', 'EPH', 'Ephesians'))

books.append( Book('50', 'PHP', 'Philippians'))
books.append( Book('51', 'COL', 'Colossians'))
books.append( Book('52', '1TH', '1 Thessalonians'))
books.append( Book('53', '2TH', '2 Thessalonians'))
books.append( Book('54', '1TI', '1 Timothy'))
books.append( Book('55', '2TI', '2 Timothy'))
books.append( Book('56', 'TIT', 'Titus'))
books.append( Book('57', 'PHL', 'Philemon'))
books.append( Book('58', 'HEB', 'Hebrews'))
books.append( Book('59', 'JAS', 'James'))

books.append( Book('60', '1PE', '1 Peter'))
books.append( Book('61', '2PE', '2 Peter'))

books.append( Book('62', '1JN', '1 John'))
books.append( Book('63', '2JN', '2 John'))
books.append( Book('64', '3JN', '3 John'))
books.append( Book('65', 'JUD', 'Jude'))
books.append( Book('66', 'REV', 'Revelation'))

##############################
#
#	Paths
#
##############################

baseDir   = os.path.dirname(os.path.abspath(__file__)) + '/'
toolsDir  = baseDir + '/../USFM-Tools'

sourceDir = baseDir + 'source'
tempDir   = tempfile.mkdtemp()
tempDir2  = tempfile.mkdtemp()

for d in [sourceDir, tempDir, tempDir2]:
    pathlib.Path(d).mkdir(parents=True, exist_ok=True) 

booklist = 'all'
config = baseDir + '/support/oeb.config'
buildId = 'oeb-working'

##############################
#
#   Update USFM and RTF
#
##############################

tags = 'us-nrsv-neut-gehenna-ioudaioi-working'
swap = ''
usfmDir   = baseDir + 'artifacts/us/usfm'
rtfDir    = baseDir + 'artifacts/us/rtf'
mdDir     = baseDir + 'artifacts/us/md'
indexDir  = baseDir + 'artifacts/us'
for d in [usfmDir, rtfDir, indexDir]: pathlib.Path(d).mkdir(parents=True, exist_ok=True) 
updateUSFM()

tags = 'cth-nrsv-neut-gehenna-ioudaioi-working'
swap = '--punctuation'
usfmDir   = baseDir + 'artifacts/cth/usfm'
rtfDir    = baseDir + 'artifacts/cth/rtf'
mdDir     = baseDir + 'artifacts/cth/md'
indexDir  = baseDir + 'artifacts/cth'
for d in [usfmDir, rtfDir, indexDir]: pathlib.Path(d).mkdir(parents=True, exist_ok=True) 
updateUSFM()

shutil.rmtree(tempDir)