From a7637dd3ba2e27db45b4b3bac0bd9c8320064c8e Mon Sep 17 00:00:00 2001 From: Sebb Date: Wed, 12 Jun 2024 00:07:01 +0100 Subject: [PATCH 01/11] Use artifacts to cache the build --- pelican/action.yml | 47 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index b312633b..3f82d01d 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -51,18 +51,53 @@ runs: # If the site uses Github Flavored Markdown, use this build branch - name: fetch and build libcmark-gfm.so if: ${{ inputs.gfm == 'true' }} + id: build_gfm shell: bash env: WORKDIR: /opt/pelican-asf # where to build GFM GFM_VERSION: '0.28.3.gfm.12' # ensure we agree with build-cmark.sh script + GITHUB_ACTION_REPO: ${{ github.action_repository }} # only works in the env context; empty for local action call + GH_TOKEN: ${{ github.token }} # needed by gh run: | + # Needs to be available to a subsequent step + echo "GFM_VERSION=${GFM_VERSION}" >>$GITHUB_ENV + if [[ -z $LIBCMARKDIR ]] # don't overwrite LIBCMARKDIR if it exists + then + # set up the GFM environment + export LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib + echo "LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib" >>$GITHUB_ENV + fi # Does the GFM build already exist? - if [[ -n $LIBCMARKDIR && -d $LIBCMARKDIR ]] + if [[ -d $LIBCMARKDIR ]] then echo "Already have GFM binary at $LIBCMARKDIR, skipping build" exit 0 # nothing more to do in this step fi + # Is there a saved artifact for the GFM build? + + REPO=${{ github.repository }} # Check calling repo for artifact + echo "Check for GFM build artifact in current repo: $REPO" + gh run download --dir ${LIBCMARKDIR} --name cmark-gfm-${GFM_VERSION} --repo $REPO || true + if [[ -d $LIBCMARKDIR ]] + then + echo "Found ${LIBCMARKDIR} in $REPO, nothing more to do!" + exit 0 # nothing more to do in this step + fi + + if [[ -n $GITHUB_ACTION_REPO ]] # Check action repo for artifact + then + echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" + gh run download --dir ${LIBCMARKDIR} --name cmark-gfm-${GFM_VERSION} --repo $GITHUB_ACTION_REPO || true + if [[ -d $LIBCMARKDIR ]] + then + echo "Found ${LIBCMARKDIR} in $GITHUB_ACTION_REPO, nothing more to do!" + exit 0 # nothing more to do in this step + fi + fi + + # GFM binary not found, need to build it { + echo "Creating GFM binary in ${LIBCMARKDIR}" # disable stdout unless debug is on if [ "${{ inputs.debug }}" == 'true' ] then @@ -74,10 +109,16 @@ runs: mkdir -p $WORKDIR pushd $WORKDIR bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION + # Message to upload step + echo "created=${LIBCMARKDIR}" >> $GITHUB_OUTPUT popd - # ensure LIBCMARKDIR is defined for subsequent steps - echo "LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib" >> $GITHUB_ENV } + - name: Save the GFM build + if: ${{ steps.build_gfm.outputs.created }} + uses: actions/upload-artifact@v4 + with: + name: cmark-gfm-${{ env.GFM_VERSION}} + path: ${{ steps.build_gfm.outputs.created }} - name: Generate website from markdown shell: bash From 8b3e1e67dd13019af8237af429c3d06b49c25bf5 Mon Sep 17 00:00:00 2001 From: Sebb Date: Fri, 14 Jun 2024 16:02:22 +0100 Subject: [PATCH 02/11] Extract artifact key name (#53) --- pelican/action.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index 3f82d01d..d3445279 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -59,13 +59,15 @@ runs: GITHUB_ACTION_REPO: ${{ github.action_repository }} # only works in the env context; empty for local action call GH_TOKEN: ${{ github.token }} # needed by gh run: | + export GFM_ARTIFACT_KEY=cmark-gfm-${GFM_VERSION} # Needs to be available to a subsequent step echo "GFM_VERSION=${GFM_VERSION}" >>$GITHUB_ENV + echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >>$GITHUB_ENV if [[ -z $LIBCMARKDIR ]] # don't overwrite LIBCMARKDIR if it exists then # set up the GFM environment export LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib - echo "LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib" >>$GITHUB_ENV + echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV fi # Does the GFM build already exist? if [[ -d $LIBCMARKDIR ]] @@ -77,7 +79,7 @@ runs: REPO=${{ github.repository }} # Check calling repo for artifact echo "Check for GFM build artifact in current repo: $REPO" - gh run download --dir ${LIBCMARKDIR} --name cmark-gfm-${GFM_VERSION} --repo $REPO || true + gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $REPO || true if [[ -d $LIBCMARKDIR ]] then echo "Found ${LIBCMARKDIR} in $REPO, nothing more to do!" @@ -87,7 +89,7 @@ runs: if [[ -n $GITHUB_ACTION_REPO ]] # Check action repo for artifact then echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" - gh run download --dir ${LIBCMARKDIR} --name cmark-gfm-${GFM_VERSION} --repo $GITHUB_ACTION_REPO || true + gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true if [[ -d $LIBCMARKDIR ]] then echo "Found ${LIBCMARKDIR} in $GITHUB_ACTION_REPO, nothing more to do!" @@ -117,7 +119,7 @@ runs: if: ${{ steps.build_gfm.outputs.created }} uses: actions/upload-artifact@v4 with: - name: cmark-gfm-${{ env.GFM_VERSION}} + name: ${{ env.GFM_ARTIFACT_KEY }} path: ${{ steps.build_gfm.outputs.created }} - name: Generate website from markdown From 07645b708e541c63fb745e3b97787c1e4f22f10d Mon Sep 17 00:00:00 2001 From: Sebb Date: Fri, 14 Jun 2024 22:13:57 +0100 Subject: [PATCH 03/11] Distinguish the artifact key from directory name --- pelican/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pelican/action.yml b/pelican/action.yml index d3445279..4063af3c 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -59,7 +59,8 @@ runs: GITHUB_ACTION_REPO: ${{ github.action_repository }} # only works in the env context; empty for local action call GH_TOKEN: ${{ github.token }} # needed by gh run: | - export GFM_ARTIFACT_KEY=cmark-gfm-${GFM_VERSION} + # The key needs to include the GFM version, but does not have to be the same as the directory name + export GFM_ARTIFACT_KEY=gfm-lib-${GFM_VERSION} # Needs to be available to a subsequent step echo "GFM_VERSION=${GFM_VERSION}" >>$GITHUB_ENV echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >>$GITHUB_ENV From 1a525a52e9469eff860d315631eaf4309ef7eab6 Mon Sep 17 00:00:00 2001 From: Sebb Date: Fri, 14 Jun 2024 22:45:12 +0100 Subject: [PATCH 04/11] Synch with main --- pelican/action.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pelican/action.yml b/pelican/action.yml index 4063af3c..d096b3b5 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -111,10 +111,14 @@ runs: # Don't pollute site checkout mkdir -p $WORKDIR pushd $WORKDIR - bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION + # build the code and define LIBCMARKDIR + bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$ + source /tmp/libcmarkdir.$$ # Message to upload step echo "created=${LIBCMARKDIR}" >> $GITHUB_OUTPUT popd + # ensure LIBCMARKDIR is defined for subsequent steps + echo "LIBCMARKDIR=${LIBCMARKDIR}" >> $GITHUB_ENV } - name: Save the GFM build if: ${{ steps.build_gfm.outputs.created }} From f11fb702e0c6d235d90a202439c34ad6bde38e1a Mon Sep 17 00:00:00 2001 From: Sebb Date: Fri, 14 Jun 2024 22:53:09 +0100 Subject: [PATCH 05/11] Keep synching with main --- pelican/plugins/age_days_lt/__init__.py | 37 ---------- pelican/plugins/age_days_lt/agedayslt.py | 24 ------- pelican/plugins/extract_toc/__init__.py | 1 - pelican/plugins/extract_toc/extract_toc.py | 65 ----------------- pelican/plugins/jinja2content/__init__.py | 1 - .../plugins/jinja2content/jinja2content.py | 67 ------------------ .../plugins/md_inline_extension/__init__.py | 1 - pelican/plugins/md_inline_extension/inline.py | 69 ------------------- .../pelican_inline_markdown_extension.py | 69 ------------------- pelican/plugins/regex_replace/__init__.py | 17 ----- .../plugins/regex_replace/regex_replace.py | 34 --------- pelican/plugins/toc.py | 4 +- pelican/plugins/toc2.py | 2 +- 13 files changed, 3 insertions(+), 388 deletions(-) delete mode 100644 pelican/plugins/age_days_lt/__init__.py delete mode 100644 pelican/plugins/age_days_lt/agedayslt.py delete mode 100644 pelican/plugins/extract_toc/__init__.py delete mode 100644 pelican/plugins/extract_toc/extract_toc.py delete mode 100644 pelican/plugins/jinja2content/__init__.py delete mode 100644 pelican/plugins/jinja2content/jinja2content.py delete mode 100644 pelican/plugins/md_inline_extension/__init__.py delete mode 100644 pelican/plugins/md_inline_extension/inline.py delete mode 100644 pelican/plugins/md_inline_extension/pelican_inline_markdown_extension.py delete mode 100644 pelican/plugins/regex_replace/__init__.py delete mode 100644 pelican/plugins/regex_replace/regex_replace.py diff --git a/pelican/plugins/age_days_lt/__init__.py b/pelican/plugins/age_days_lt/__init__.py deleted file mode 100644 index dd32091a..00000000 --- a/pelican/plugins/age_days_lt/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -The age_in_days plugin adds a Jinja test, age_days_lt. - -It is intended to be used in Pelican templates like this to select articles newer than 90 days: - - {% for article in (articles | selectattr("date", "age_days_lt", 90) ) %} - ... - {% endif %} -""" -from pelican import signals -from . import agedayslt - -def add_test(pelican): - """Add age_days_lt test to Pelican.""" - pelican.env.tests.update({'age_days_lt': agedayslt.age_days_lt}) - - -def register(): - """Plugin registration.""" - signals.generator_init.connect(add_test) diff --git a/pelican/plugins/age_days_lt/agedayslt.py b/pelican/plugins/age_days_lt/agedayslt.py deleted file mode 100644 index c191df56..00000000 --- a/pelican/plugins/age_days_lt/agedayslt.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from datetime import datetime - -def age_days_lt(dt, days): - """Return true if a number of days since 'dt' < 'days'""" - now = datetime.now(dt.tzinfo) - delta = now - dt - return delta.days < days diff --git a/pelican/plugins/extract_toc/__init__.py b/pelican/plugins/extract_toc/__init__.py deleted file mode 100644 index 52c57781..00000000 --- a/pelican/plugins/extract_toc/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .extract_toc import * diff --git a/pelican/plugins/extract_toc/extract_toc.py b/pelican/plugins/extract_toc/extract_toc.py deleted file mode 100644 index 210bac84..00000000 --- a/pelican/plugins/extract_toc/extract_toc.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Extract Table of Content -======================== -A Pelican plugin to extract table of contents (ToC) from `article.content` and -place it in its own `article.toc` variable for use in templates. -""" - -from os import path -from bs4 import BeautifulSoup -from pelican import signals, readers, contents -import logging - -logger = logging.getLogger(__name__) - - -def extract_toc(content): - if isinstance(content, contents.Static): - return - - soup = BeautifulSoup(content._content, 'html.parser') # pylint: disable=protected-access - filename = content.source_path - extension = path.splitext(filename)[1][1:] - toc = None - - # default Markdown reader - if not toc and readers.MarkdownReader.enabled and extension in readers.MarkdownReader.file_extensions: - toc = soup.find('div', class_='toc') - if toc: - toc.extract() - if len(toc.find_next('ul').find_all('li')) == 0: - toc = None - - # default reStructuredText reader - if not toc and readers.RstReader.enabled and extension in readers.RstReader.file_extensions: - toc = soup.find('div', class_='contents topic') - if toc: - toc.extract() - tag = BeautifulSoup(str(toc), 'html.parser') - tag.div['class'] = 'toc' - tag.div['id'] = '' - p = tag.find('p', class_='topic-title first') - if p: - p.extract() - toc = tag - - # Pandoc reader (markdown and other formats) - if 'pandoc_reader' in content.settings['PLUGINS']: - try: - from pandoc_reader import PandocReader - except ImportError: - PandocReader = False - if not toc and PandocReader and PandocReader.enabled and extension in PandocReader.file_extensions: - toc = soup.find('nav', id='TOC') - - if toc: - toc.extract() - content._content = soup.decode() # pylint: disable=protected-access - content.toc = toc.decode() - if content.toc.startswith(''): - content.toc = content.toc[12:-14] - - -def register(): - signals.content_object_init.connect(extract_toc) diff --git a/pelican/plugins/jinja2content/__init__.py b/pelican/plugins/jinja2content/__init__.py deleted file mode 100644 index de025c4b..00000000 --- a/pelican/plugins/jinja2content/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .jinja2content import * diff --git a/pelican/plugins/jinja2content/jinja2content.py b/pelican/plugins/jinja2content/jinja2content.py deleted file mode 100644 index 44ff18ba..00000000 --- a/pelican/plugins/jinja2content/jinja2content.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -jinja2content.py ----------------- -Pelican plugin that processes Markdown files as jinja templates. -""" - -from jinja2 import Environment, FileSystemLoader, ChoiceLoader -import os -from pelican import signals -from pelican.readers import MarkdownReader, HTMLReader, RstReader -from pelican.utils import pelican_open -from tempfile import NamedTemporaryFile - -class JinjaContentMixin: - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # will look first in 'JINJA2CONTENT_TEMPLATES', by default the - # content root path, then in the theme's templates - local_dirs = self.settings.get('JINJA2CONTENT_TEMPLATES', ['.']) - local_dirs = [os.path.join(self.settings['PATH'], folder) - for folder in local_dirs] - theme_dir = os.path.join(self.settings['THEME'], 'templates') - - loaders = [FileSystemLoader(_dir) for _dir - in local_dirs + [theme_dir]] - if 'JINJA_ENVIRONMENT' in self.settings: # pelican 3.7 - jinja_environment = self.settings['JINJA_ENVIRONMENT'] - else: - jinja_environment = { - 'trim_blocks': True, - 'lstrip_blocks': True, - 'extensions': self.settings['JINJA_EXTENSIONS'] - } - self.env = Environment( - loader=ChoiceLoader(loaders), - **jinja_environment) - - - def read(self, source_path): - with pelican_open(source_path) as text: - text = self.env.from_string(text).render() - - with NamedTemporaryFile(delete=False) as f: - f.write(text.encode()) - f.close() - content, metadata = super().read(f.name) - os.unlink(f.name) - return content, metadata - - -class JinjaMarkdownReader(JinjaContentMixin, MarkdownReader): - pass - -class JinjaRstReader(JinjaContentMixin, RstReader): - pass - -class JinjaHTMLReader(JinjaContentMixin, HTMLReader): - pass - -def add_reader(readers): - for Reader in [JinjaMarkdownReader, JinjaRstReader, JinjaHTMLReader]: - for ext in Reader.file_extensions: - readers.reader_classes[ext] = Reader - -def register(): - signals.readers_init.connect(add_reader) diff --git a/pelican/plugins/md_inline_extension/__init__.py b/pelican/plugins/md_inline_extension/__init__.py deleted file mode 100644 index 2453fe9d..00000000 --- a/pelican/plugins/md_inline_extension/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .inline import * diff --git a/pelican/plugins/md_inline_extension/inline.py b/pelican/plugins/md_inline_extension/inline.py deleted file mode 100644 index aab0736e..00000000 --- a/pelican/plugins/md_inline_extension/inline.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Markdown Inline Extension For Pelican -===================================== -Extends Pelican's Markdown module -and allows for customized inline HTML -""" - -# mypy: ignore-errors - -import sys - -from pelican import signals - -try: - from . pelican_inline_markdown_extension import PelicanInlineMarkdownExtension -except ImportError as e: - PelicanInlineMarkdownExtension = None - print("\nMarkdown is not installed - inline Markdown extension disabled\n") - -def process_settings(pelicanobj): - """Sets user specified settings (see README for more details)""" - - # Default settings - inline_settings = {} - inline_settings['config'] = {'[]':('', 'pelican-inline')} - - # Get the user specified settings - try: - settings = pelicanobj.settings['MD_INLINE'] - except Exception: # TODO: narrow further to expected Exceptions - settings = None - - # If settings have been specified, add them to the config - if isinstance(settings, dict): - inline_settings['config'].update(settings) - - return inline_settings - -def inline_markdown_extension(pelicanobj, config): - """Instantiates a customized Markdown extension""" - - # Instantiate Markdown extension and append it to the current extensions - try: - if isinstance(pelicanobj.settings.get('MD_EXTENSIONS'), list): # pelican 3.6.3 and earlier - pelicanobj.settings['MD_EXTENSIONS'].append(PelicanInlineMarkdownExtension(config)) - else: - pelicanobj.settings['MARKDOWN'].setdefault('extensions', []).append(PelicanInlineMarkdownExtension(config)) - except Exception: # TODO: narrow further to expected Exceptions - sys.excepthook(*sys.exc_info()) - sys.stderr.write("\nError - the pelican Markdown extension failed to configure. Inline Markdown extension is non-functional.\n") - sys.stderr.flush() - -def pelican_init(pelicanobj): - """Loads settings and instantiates the Python Markdown extension""" - - # If there was an error loading Markdown, then do not process any further - if not PelicanInlineMarkdownExtension: - return - - # Process settings - config = process_settings(pelicanobj) - - # Configure Markdown Extension - inline_markdown_extension(pelicanobj, config) - -def register(): - """Plugin registration""" - signals.initialized.connect(pelican_init) diff --git a/pelican/plugins/md_inline_extension/pelican_inline_markdown_extension.py b/pelican/plugins/md_inline_extension/pelican_inline_markdown_extension.py deleted file mode 100644 index 7ef526b5..00000000 --- a/pelican/plugins/md_inline_extension/pelican_inline_markdown_extension.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Pelican Inline Markdown Extension -================================== -An extension for the Python Markdown module that enables -the Pelican Python static site generator to add inline patterns. -""" - -# pylint: skip-file - -import markdown -import re - -class PelicanInlineMarkdownExtensionPattern(markdown.inlinepatterns.Pattern): - """Inline Markdown processing""" - - def __init__(self, pelican_markdown_extension, tag, pattern): - super(PelicanInlineMarkdownExtensionPattern,self).__init__(pattern) - self.tag = tag - self.config = pelican_markdown_extension.getConfig('config') - - def handleMatch(self, m): - node = markdown.util.etree.Element(self.tag) - tag_attributes = self.config.get(m.group('prefix'), ('', 'pelican-inline')) - tag_class = 'pelican-inline' # default class - tag_style = '' # default is for no styling - - if isinstance(tag_attributes, tuple): - tag_style = tag_attributes[0] - tag_class = tag_attributes[1] if len(tag_attributes) > 1 else '' - elif isinstance(tag_attributes, str): - tag_class = tag_attributes - - if tag_class != '': - node.set('class', tag_class) - if tag_style!= '': - node.set('style', tag_style) - - node.text = markdown.util.AtomicString(m.group('text')) - - return node - -class PelicanInlineMarkdownExtension(markdown.Extension): - """A Markdown extension enabling processing in Markdown for Pelican""" - def __init__(self, config): - - try: - # Needed for Markdown versions >= 2.5 - self.config['config'] = ['{}', 'config for markdown extension'] - super(PelicanInlineMarkdownExtension,self).__init__(**config) - except AttributeError: - # Markdown versions < 2.5 - config['config'] = [config['config'], 'config for markdown extension'] - super(PelicanInlineMarkdownExtension, self).__init__(config) - - # N.B. md_globals parameter was dropped: https://github.com/Python-Markdown/markdown/pull/697 - def extendMarkdown(self, md): - # Regex to detect mathjax - config = self.getConfig('config') - patterns = [] - - # The following mathjax settings can be set via the settings dictionary - for key in config: - patterns.append(re.escape(key)) - - inline_regex = r'(?P%s)(?P.+?)\2' % ('|'.join(patterns)) - - # Process after escapes - md.inlinePatterns.add('texthighlight_inlined', PelicanInlineMarkdownExtensionPattern(self, 'span', inline_regex), '>emphasis2') diff --git a/pelican/plugins/regex_replace/__init__.py b/pelican/plugins/regex_replace/__init__.py deleted file mode 100644 index f97c8a3e..00000000 --- a/pelican/plugins/regex_replace/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from .regex_replace import * diff --git a/pelican/plugins/regex_replace/regex_replace.py b/pelican/plugins/regex_replace/regex_replace.py deleted file mode 100644 index bbe544b5..00000000 --- a/pelican/plugins/regex_replace/regex_replace.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Markdown regex_replace filter for pelican -""" -from pelican import signals -import re - -# Custom filter method -def regex_replace(s, find, replace): - return re.sub(find, replace, s) - -def add_filter(pelican): - """Add filter to Pelican.""" - pelican.env.filters.update({'regex_replace': regex_replace}) - -def register(): - """Plugin registration.""" - signals.generator_init.connect(add_filter) diff --git a/pelican/plugins/toc.py b/pelican/plugins/toc.py index 81157e4e..be13485f 100644 --- a/pelican/plugins/toc.py +++ b/pelican/plugins/toc.py @@ -127,7 +127,7 @@ def generate_toc(content): raise e # Find TOC tag - tocTag = soup.find('p', text = '[TOC]') + tocTag = soup.find('p', text='[TOC]') if tocTag: for header in tocTag.findAllNext(header_re): settoc = True @@ -139,7 +139,7 @@ def generate_toc(content): tree_string = '{}'.format(tree) tree_soup = BeautifulSoup(tree_string, 'html.parser') content.toc = tree_soup.decode(formatter='html') - itoc = soup.find('p', text = '[TOC]') + itoc = soup.find('p', text='[TOC]') if itoc: itoc.replaceWith(tree_soup) diff --git a/pelican/plugins/toc2.py b/pelican/plugins/toc2.py index d3287470..0779bda0 100644 --- a/pelican/plugins/toc2.py +++ b/pelican/plugins/toc2.py @@ -138,7 +138,7 @@ def generate_toc(content): if settoc: print("Generating ToC for %s" % content.slug) tree_string = '{}'.format(tree) - print("ToC: %s" % tree_string) + # print("ToC: %s" % tree_string) tree_soup = BeautifulSoup(tree_string, 'html.parser') content.toc = tree_soup.decode(formatter='html') itoc = soup.find('p', text='[TOC]') From 4d99cd3d77a7e9a15c8759e261c700cc24d6f55e Mon Sep 17 00:00:00 2001 From: Sebb Date: Fri, 14 Jun 2024 23:16:17 +0100 Subject: [PATCH 06/11] Don't assume library name (#55) --- pelican/action.yml | 3 ++- pelican/build-cmark.sh | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index d096b3b5..eea95f68 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -67,7 +67,8 @@ runs: if [[ -z $LIBCMARKDIR ]] # don't overwrite LIBCMARKDIR if it exists then # set up the GFM environment - export LIBCMARKDIR=${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib + NOBUILD=1 bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir1.$$ + source /tmp/libcmarkdir1.$$ echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV fi # Does the GFM build already exist? diff --git a/pelican/build-cmark.sh b/pelican/build-cmark.sh index c9296dc8..8d8a9001 100644 --- a/pelican/build-cmark.sh +++ b/pelican/build-cmark.sh @@ -30,6 +30,13 @@ LOCAL="${TARDIR}/cmark-gfm.$VERSION.orig.tar.gz" # WARNING: this must agree with the parent directory in the tar file or the build will fail EXTRACTED_AS="cmark-gfm-$VERSION" +LIBCMARKDIR="$(pwd)/$EXTRACTED_AS/lib" + +# Allow caller to find out library directory without needing to build first +if [[ -n $NOBUILD ]] +then + echo "export LIBCMARKDIR='$LIBCMARKDIR'" +fi # Follow redirects, and place the result into known name $LOCAL if [ -f "$LOCAL" ]; then @@ -63,4 +70,4 @@ popd >/dev/null # ls -laF "$EXTRACTED_AS/lib/" # Provide a handy line for copy/paste. -echo "export LIBCMARKDIR='$(pwd)/$EXTRACTED_AS/lib'" +echo "export LIBCMARKDIR='$LIBCMARKDIR'" From a6bafc0e6197b806d1efcd4e5e451d531eafe419 Mon Sep 17 00:00:00 2001 From: Sebb Date: Sat, 15 Jun 2024 13:54:42 +0100 Subject: [PATCH 07/11] Opps - actually skip the build (#57) --- pelican/build-cmark.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pelican/build-cmark.sh b/pelican/build-cmark.sh index 8d8a9001..737991c7 100644 --- a/pelican/build-cmark.sh +++ b/pelican/build-cmark.sh @@ -36,6 +36,7 @@ LIBCMARKDIR="$(pwd)/$EXTRACTED_AS/lib" if [[ -n $NOBUILD ]] then echo "export LIBCMARKDIR='$LIBCMARKDIR'" + exit # skip the build fi # Follow redirects, and place the result into known name $LOCAL From 3ebd89e274a309221330add7f157c74e3809846c Mon Sep 17 00:00:00 2001 From: Sebb Date: Sat, 15 Jun 2024 14:28:16 +0100 Subject: [PATCH 08/11] Don't cache artifacts in user repositories (#58) Use artifact from action repo only --- pelican/action.yml | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index eea95f68..ebf3e335 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -56,10 +56,13 @@ runs: env: WORKDIR: /opt/pelican-asf # where to build GFM GFM_VERSION: '0.28.3.gfm.12' # ensure we agree with build-cmark.sh script - GITHUB_ACTION_REPO: ${{ github.action_repository }} # only works in the env context; empty for local action call + # action_repository only works in the env context; empty for local action call + # it is always empty for local invocation, in which case use the current repo + GITHUB_ACTION_REPO: ${{ github.action_repository || github.repository }} GH_TOKEN: ${{ github.token }} # needed by gh run: | # The key needs to include the GFM version, but does not have to be the same as the directory name + # must agree with the definition in build-actions.yml export GFM_ARTIFACT_KEY=gfm-lib-${GFM_VERSION} # Needs to be available to a subsequent step echo "GFM_VERSION=${GFM_VERSION}" >>$GITHUB_ENV @@ -77,28 +80,16 @@ runs: echo "Already have GFM binary at $LIBCMARKDIR, skipping build" exit 0 # nothing more to do in this step fi - # Is there a saved artifact for the GFM build? - REPO=${{ github.repository }} # Check calling repo for artifact - echo "Check for GFM build artifact in current repo: $REPO" - gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $REPO || true + # Is there a saved artifact for the GFM build? + echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" + gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true if [[ -d $LIBCMARKDIR ]] then - echo "Found ${LIBCMARKDIR} in $REPO, nothing more to do!" + echo "Downloaded to ${LIBCMARKDIR} from $GITHUB_ACTION_REPO, nothing more to do!" exit 0 # nothing more to do in this step fi - if [[ -n $GITHUB_ACTION_REPO ]] # Check action repo for artifact - then - echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" - gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true - if [[ -d $LIBCMARKDIR ]] - then - echo "Found ${LIBCMARKDIR} in $GITHUB_ACTION_REPO, nothing more to do!" - exit 0 # nothing more to do in this step - fi - fi - # GFM binary not found, need to build it { echo "Creating GFM binary in ${LIBCMARKDIR}" @@ -115,18 +106,10 @@ runs: # build the code and define LIBCMARKDIR bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$ source /tmp/libcmarkdir.$$ - # Message to upload step - echo "created=${LIBCMARKDIR}" >> $GITHUB_OUTPUT popd # ensure LIBCMARKDIR is defined for subsequent steps echo "LIBCMARKDIR=${LIBCMARKDIR}" >> $GITHUB_ENV } - - name: Save the GFM build - if: ${{ steps.build_gfm.outputs.created }} - uses: actions/upload-artifact@v4 - with: - name: ${{ env.GFM_ARTIFACT_KEY }} - path: ${{ steps.build_gfm.outputs.created }} - name: Generate website from markdown shell: bash From 82760d9a77298b6f6f0b5039e9c4dfb1e92daf37 Mon Sep 17 00:00:00 2001 From: Sebb Date: Sat, 15 Jun 2024 15:11:45 +0100 Subject: [PATCH 09/11] Download the artifact to the work dir (#59) Ensure LIBCMARKDIR uses WORKDIR --- pelican/action.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index ebf3e335..0c19ec07 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -70,9 +70,13 @@ runs: if [[ -z $LIBCMARKDIR ]] # don't overwrite LIBCMARKDIR if it exists then # set up the GFM environment - NOBUILD=1 bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir1.$$ - source /tmp/libcmarkdir1.$$ - echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV + # Don't pollute site checkout + mkdir -p $WORKDIR + pushd $WORKDIR >/dev/null # ensure LIBCMARKDIR is under WORKDIR + NOBUILD=1 bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir1.$$ + source /tmp/libcmarkdir1.$$ + echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV + popd >/dev/null fi # Does the GFM build already exist? if [[ -d $LIBCMARKDIR ]] From f5246ff0fb425677b5109c7540d2276d34fcd5e2 Mon Sep 17 00:00:00 2001 From: Sebb Date: Sun, 16 Jun 2024 00:56:54 +0100 Subject: [PATCH 10/11] Artifact caching (#60) --- pelican/Dockerfile | 20 +++++++++---- pelican/action.yml | 39 +++++++++--------------- pelican/build-cmark.sh | 67 +++++++++++++++--------------------------- 3 files changed, 53 insertions(+), 73 deletions(-) diff --git a/pelican/Dockerfile b/pelican/Dockerfile index 931b2f49..b895c592 100644 --- a/pelican/Dockerfile +++ b/pelican/Dockerfile @@ -1,11 +1,16 @@ # Settings # ======== # Use the Python version as installed on CI pelican builders (2023-06-02) -ARG PYTHON_VERSION=3.8.10 -ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy below +ARG PYTHON_VERSION=3.8.10 # must agree with the copy below + +# Note: ARG scope ends at the FROM statement, so must be repeated if necessary # Build cmake-gfm FROM python:${PYTHON_VERSION}-slim-buster as pelican-asf +#======================================================= + +ARG PYTHON_VERSION=3.8.10 # must agree with the copy above +ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy below RUN apt update && apt upgrade -y RUN apt install curl cmake build-essential -y @@ -15,11 +20,15 @@ WORKDIR /opt/pelican-asf # Copy only what we need to build cmark-gfm COPY build-cmark.sh bin/build-cmark.sh -# build it -RUN bash bin/build-cmark.sh ${GFM_VERSION} +# build GFM +# Must agree with definition below +ENV LIBCMARKDIR /opt/gfm-${GFM_VERSION}/lib +RUN mkdir -p ${LIBCMARKDIR} +RUN bash bin/build-cmark.sh ${GFM_VERSION} ${LIBCMARKDIR} # rebase the image to save on image size FROM python:${PYTHON_VERSION}-slim-buster +#======================================================= # Use the Pelican version as installed on CI pelican builders (2023-06-02) ARG PELICAN_VERSION=4.5.4 @@ -27,7 +36,8 @@ ARG GFM_VERSION=0.28.3.gfm.12 # must agree with copy above # Where we put GFM and the plugins ENV WORKDIR /opt/pelican-asf -ENV LIBCMARKDIR ${WORKDIR}/cmark-gfm-${GFM_VERSION}/lib +# Must agree with definition above +ENV LIBCMARKDIR /opt/gfm-${GFM_VERSION}/lib RUN apt update && apt upgrade -y diff --git a/pelican/action.yml b/pelican/action.yml index 0c19ec07..934b4f9e 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -54,32 +54,26 @@ runs: id: build_gfm shell: bash env: - WORKDIR: /opt/pelican-asf # where to build GFM GFM_VERSION: '0.28.3.gfm.12' # ensure we agree with build-cmark.sh script # action_repository only works in the env context; empty for local action call # it is always empty for local invocation, in which case use the current repo GITHUB_ACTION_REPO: ${{ github.action_repository || github.repository }} GH_TOKEN: ${{ github.token }} # needed by gh run: | - # The key needs to include the GFM version, but does not have to be the same as the directory name - # must agree with the definition in build-actions.yml + # The key needs to include the GFM version, but is otherwise arbitrary. + # It must agree with the definition in build-actions.yml export GFM_ARTIFACT_KEY=gfm-lib-${GFM_VERSION} - # Needs to be available to a subsequent step - echo "GFM_VERSION=${GFM_VERSION}" >>$GITHUB_ENV - echo "GFM_ARTIFACT_KEY=${GFM_ARTIFACT_KEY}" >>$GITHUB_ENV - if [[ -z $LIBCMARKDIR ]] # don't overwrite LIBCMARKDIR if it exists + + if [[ -z $LIBCMARKDIR ]] # define LIBCMARKDIR if it is not already then # set up the GFM environment - # Don't pollute site checkout - mkdir -p $WORKDIR - pushd $WORKDIR >/dev/null # ensure LIBCMARKDIR is under WORKDIR - NOBUILD=1 bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir1.$$ - source /tmp/libcmarkdir1.$$ - echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV - popd >/dev/null + export LIBCMARKDIR=/opt/pelican-asf/gfm-${GFM_VERSION} # arbitrary, but should contain version + mkdir -p $LIBCMARKDIR + echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV # needed for the build step fi + ls -l $LIBCMARKDIR # Does the GFM build already exist? - if [[ -d $LIBCMARKDIR ]] + if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]] then echo "Already have GFM binary at $LIBCMARKDIR, skipping build" exit 0 # nothing more to do in this step @@ -88,7 +82,8 @@ runs: # Is there a saved artifact for the GFM build? echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true - if [[ -d $LIBCMARKDIR ]] + ls -l $LIBCMARKDIR + if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]] then echo "Downloaded to ${LIBCMARKDIR} from $GITHUB_ACTION_REPO, nothing more to do!" exit 0 # nothing more to do in this step @@ -104,16 +99,10 @@ runs: else exec >/dev/null fi - # Don't pollute site checkout - mkdir -p $WORKDIR - pushd $WORKDIR - # build the code and define LIBCMARKDIR - bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION | grep "export LIBCMARKDIR" >/tmp/libcmarkdir.$$ - source /tmp/libcmarkdir.$$ - popd - # ensure LIBCMARKDIR is defined for subsequent steps - echo "LIBCMARKDIR=${LIBCMARKDIR}" >> $GITHUB_ENV + # build the code and define LIBCMARKDIR under $WORKDIR + bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION $LIBCMARKDIR } + ls -l $LIBCMARKDIR - name: Generate website from markdown shell: bash diff --git a/pelican/build-cmark.sh b/pelican/build-cmark.sh index 737991c7..aa50f086 100644 --- a/pelican/build-cmark.sh +++ b/pelican/build-cmark.sh @@ -1,15 +1,15 @@ #!/bin/bash # -# Build the cmark-gfm library and extensions within CURRENT DIRECTORY. +# Build the cmark-gfm library and extensions in a temporary directory # -# The binary output will be under: cmark-gfm-$VERSION/lib +# The binary output will be under: LIBCMARKDIR # # USAGE: -# $ build-cmark.sh [ VERSION [ TARDIR ] ] +# $ build-cmark.sh VERSION LIBCMARKDIR [TARFILE] # -# VERSION: defaults to 0.28.3.gfm.12 -# TARDIR: where to find a downloaded/cached tarball of the cmark -# code, or where to place a tarball +# VERSION: e.g. 0.28.3.gfm.12 +# LIBCMARKDIR: where to put the binary library files +# TARFILE: local copy of the tarfile; must be for the correct version! (optional) # # Echo all of our steps if DEBUG_STEPS is set @@ -17,41 +17,30 @@ test -n "$DEBUG_STEPS" && set -x set -e # early exit if any step fails -#VERSION=0.28.3.gfm.20 ### not yet -VERSION=0.28.3.gfm.12 -if [ "$1" != "" ]; then VERSION="$1"; fi - -# The tarball exists here, or will be downloaded here. -TARDIR="." -if [ "$2" != "" ]; then TARDIR="$2"; fi +VERSION=${1:?version} +LIBCMARKDIR=${2:?library output} +TARFILE=$3 ARCHIVES="https://github.com/github/cmark-gfm/archive/refs/tags" -LOCAL="${TARDIR}/cmark-gfm.$VERSION.orig.tar.gz" +TARNAME="cmark-gfm.$VERSION.orig.tar.gz" +TARDIR="cmark-gfm-$VERSION" -# WARNING: this must agree with the parent directory in the tar file or the build will fail -EXTRACTED_AS="cmark-gfm-$VERSION" -LIBCMARKDIR="$(pwd)/$EXTRACTED_AS/lib" +# Work in a temporary directory +TEMP=$(mktemp -d) -# Allow caller to find out library directory without needing to build first -if [[ -n $NOBUILD ]] +if [[ -f $TARFILE ]] then - echo "export LIBCMARKDIR='$LIBCMARKDIR'" - exit # skip the build -fi - -# Follow redirects, and place the result into known name $LOCAL -if [ -f "$LOCAL" ]; then - echo "Using cached tarball: ${LOCAL}" >&2 + echo "Found tar!" + cp $TARFILE $TEMP # do this before cd to allow for relative paths + cd $TEMP else - echo "Fetching $VERSION from cmark archives" >&2 - curl -sSL --fail -o "$LOCAL" "$ARCHIVES/$VERSION.tar.gz" + cd $TEMP + echo "Fetching $VERSION from cmark archives" >&2 + curl -sSL --fail -o "$TARNAME" "$ARCHIVES/$VERSION.tar.gz" fi -# Clean anything old, then extract and build. -### somebody smart could peek into the .tgz. ... MEH -if [ -d "$EXTRACTED_AS" ]; then rm -r "$EXTRACTED_AS"; fi -tar xzf "$LOCAL" -pushd "$EXTRACTED_AS" >/dev/null +tar xzf "$TARNAME" +pushd "$TARDIR" >/dev/null mkdir build pushd build >/dev/null cmake --version >&2 @@ -61,14 +50,6 @@ pushd "$EXTRACTED_AS" >/dev/null } > build.log popd >/dev/null - mkdir lib - cp -Pp build/src/lib* lib/ - cp -Pp build/extensions/lib* lib/ + cp -Pp build/src/lib* ${LIBCMARKDIR}/ + cp -Pp build/extensions/lib* ${LIBCMARKDIR}/ popd >/dev/null - -# These files/dir may need a reference with LD_LIBRARY_PATH. -# gfm.py wants this lib/ in LIBCMARKDIR. -# ls -laF "$EXTRACTED_AS/lib/" - -# Provide a handy line for copy/paste. -echo "export LIBCMARKDIR='$LIBCMARKDIR'" From 79a054353328d361b3d028becd216ba4f350e77e Mon Sep 17 00:00:00 2001 From: Sebb Date: Sun, 16 Jun 2024 01:01:30 +0100 Subject: [PATCH 11/11] Drop debug (#61) --- pelican/action.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pelican/action.yml b/pelican/action.yml index 934b4f9e..d1832d4a 100644 --- a/pelican/action.yml +++ b/pelican/action.yml @@ -71,7 +71,7 @@ runs: mkdir -p $LIBCMARKDIR echo "LIBCMARKDIR=${LIBCMARKDIR}" >>$GITHUB_ENV # needed for the build step fi - ls -l $LIBCMARKDIR + # Does the GFM build already exist? if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]] then @@ -82,7 +82,6 @@ runs: # Is there a saved artifact for the GFM build? echo "Check for GFM build artifact in action repo: $GITHUB_ACTION_REPO" gh run download --dir ${LIBCMARKDIR} --name ${GFM_ARTIFACT_KEY} --repo $GITHUB_ACTION_REPO || true - ls -l $LIBCMARKDIR if [[ -f $LIBCMARKDIR/libcmark-gfm.so ]] then echo "Downloaded to ${LIBCMARKDIR} from $GITHUB_ACTION_REPO, nothing more to do!" @@ -102,7 +101,6 @@ runs: # build the code and define LIBCMARKDIR under $WORKDIR bash ${{ github.action_path }}/build-cmark.sh $GFM_VERSION $LIBCMARKDIR } - ls -l $LIBCMARKDIR - name: Generate website from markdown shell: bash