Merge pull request #38 from OSAS/nikola_support

add Nikola builder support

Author: duck-rh

README.md

@@ -6,6 +6,7 @@ Currently this role supports the following web generators:
 * Ascii Binder (http://asciibinder.org/)
 * [Jekyll](https://jekyllrb.com/)
 * [Middleman](https://middlemanapp.com/)
+* [Nikola](https://getnikola.com/)
 * Planet (using Planet Venus)
 
 # Example
@@ -58,6 +59,13 @@ the time between automated rebuild attempts if nothing changed, expressed in hou
 
 This feature is disabled on containers.
 
+# Builder deployment specific parameters
+
+Sometimes a builder deployment really needs to be adapted because there is no way to do so later in the build configuration.
+
+- **nikola_theme**: theme name to install using Nikola's repository
+                    (the configuration file is not parsed, you need to maintain both parameters in sync)
+
 # Debug the build
 
 In order to debug a non working build, the easiest is to connect to the

molecule/_resources/Dockerfile.j2

@@ -6,8 +6,10 @@ FROM {{ item.registry.url }}/{{ item.image }}
 FROM {{ item.image }}
 {% endif %}
 
+# on Red Hat systems using dnf, Python 3 is already installed
+# the package is versioned and there is no 'python3' dependency package, so better do nothing
 RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get upgrade -y && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
-    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python3-devel python3-dnf bash && dnf clean all; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python3-devel python3-dnf bash && dnf clean all; \
     elif [ $(command -v yum) ]; then yum makecache fast && yum update -y && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
     elif [ $(command -v zypper) ]; then zypper refresh && zypper update -y && zypper install -y python sudo bash python-xml && zypper clean -a; \
     elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; fi

molecule/_resources/prepare.yml

@@ -10,7 +10,7 @@
           - hostname
           - glibc-all-langpacks
         state: installed
-      when: ansible_distribution == 'Fedora'
+      when: ansible_distribution == 'Fedora' or (ansible_os_family == "RedHat" and ansible_distribution_major_version|int >= 8)
     # to ensure having a real FQDN
     # (dots are not allowed in LXD container names, thus sticking to short names)
     - name: set proper hostname

molecule/base_molecule.yml

@@ -27,6 +27,8 @@ provisioner:
                 aliases:
                   # allow access via short DNS name
                   - "{{ item.name }}"
+        # used to switch the testing branch
+        tests_is_container: False
     host_vars:
       ansible-test-builder:
       ansible-test-web:

molecule/nikola-el8/molecule.yml

@@ -0,0 +1,24 @@
+---
+scenario:
+  name: nikola-el8
+platforms:
+  - name: ansible-test-builder
+    # LXD
+    source:
+      alias: 'centos/8/amd64'
+    # Docker
+    image: 'centos:8'
+  # this one is always the same
+  - name: ansible-test-web
+    # LXD
+    source:
+      alias: 'centos/8/amd64'
+    # Docker
+    image: 'centos:8'
+provisioner:
+  inventory:
+    host_vars:
+      ansible-test-builder:
+        builder: nikola
+        # known working website
+        git_url: https://pagure.io/koji-site.git

molecule/nikola-el8/tests/test_default.py

@@ -0,0 +1 @@
+../../_resources/tests/test_default.py

molecule/planet-el7/molecule.yml

@@ -21,4 +21,4 @@ provisioner:
       ansible-test-builder:
         builder: planet
         # known working website
-        git_url: https://github.com/redhat-openstack/website.git
+        git_url: https://gitlab.com/osas/ansible-role-web_builder_test_planet.git

tasks/main.yml

@@ -6,6 +6,14 @@
     - "{{ ansible_distribution }}.yml"
     - "{{ ansible_os_family }}.yml"
 
+- name: "Enable PowerTools RPM Repository"
+  ini_file:
+    path: /etc/yum.repos.d/CentOS-PowerTools.repo
+    section: PowerTools
+    option: enabled
+    value: '1'
+  when: ansible_os_family == "RedHat" and ansible_distribution_major_version|int >= 8
+
 - name: Install common and builder-specific packages
   package:
     name: "{{ package_list['common'] + package_list[builder] }}"
@@ -24,7 +32,7 @@
     checkout_dir: "/srv/builder/{{ builder_name }}"
 
 - name: Deploy the build script
-  copy:
+  template:
     dest: /usr/local/bin/build_deploy.py
     src: build_deploy.py
     mode: 0755
@@ -95,6 +103,17 @@
   changed_when: False
   when: gemfile_st.stat.exists
 
+# after workdir is setup as it might use it
+- name: "Extra builder setup"
+  command: "{{ item }}"
+  args:
+    chdir: "{{ checkout_dir }}"
+  become: yes
+  become_user: '{{ builder_username }}'
+  become_method: 'su'
+  changed_when: False
+  loop: "{{ builder_setup[builder] | default([]) }}"
+
 - name: Clear RSYNC URL
   set_fact:
     rsync_url: "nosync"
@@ -122,9 +141,9 @@
 
 - name: "Schedule build and sync"
   include_tasks: schedule_build_and_sync.yml
-  when: ansible_env.container is undefined
+  when: (tests_is_container is defined) | ternary(not tests_is_container, ansible_env.container is undefined)
 
 - name: "Build now (container)"
   include_tasks: build.yml
-  when: ansible_env.container is defined and builder_container_build_now|bool
+  when: ((tests_is_container is defined) | ternary(tests_is_container, ansible_env.container is defined)) and builder_container_build_now|bool
 

files/build_deploy.py renamed to templates/build_deploy.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python{{ ((ansible_os_family == "RedHat" and ansible_distribution_major_version|int >= 8) or (ansible_os_family == "Debian" and ansible_distribution_major_version|int >= 10)) | ternary('3', '') }}
 #
 # Copyright (c) 2015 Michael Scherer <[email protected]>
 #
@@ -80,6 +80,12 @@
         'build_command': ['/srv/builder/planet-venus/planet.py', '-v', 'planet.ini'],
         'build_subdir': 'build',
         'deploy_command': None
+    },
+    'nikola': {
+        'build_env': {},
+        'build_command': ['../.local/bin/nikola', 'build'],
+        'build_subdir': 'output',
+        'deploy_command': None
     }
 }
 
@@ -96,47 +102,47 @@ def log_print(message):
 
 def debug_print(message):
     if args.debug:
-        print message
+        print(message)
     log_print(message)
 
 
 def refresh_checkout(checkout_dir):
     os.chdir(checkout_dir)
     try:
         result = subprocess.check_output(['git', 'fetch', '-q'], stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError, C:
+    except subprocess.CalledProcessError as C:
         notify_error('setup', C.output)
-    debug_print(result)
+    debug_print(result.decode())
 
 
 def get_last_commit(checkout_dir):
     os.chdir(checkout_dir)
     try:
         r = subprocess.check_output(['git', 'ls-remote', '-q', '.',
                                      'refs/remotes/origin/%s' % config['git_version']])
-    except subprocess.CalledProcessError, C:
+    except subprocess.CalledProcessError as C:
         notify_error('setup', C.output)
-    return r.split()[0]
+    return r.decode().split()[0]
 
 
 def get_last_commit_submodule(checkout_dir, submodule):
     os.chdir("{}/{}/".format(checkout_dir, submodule))
     try:
         r = subprocess.check_output(['git', 'ls-remote', '-q', '.',
                                      'refs/remotes/origin/HEAD'])
-    except subprocess.CalledProcessError, C:
+    except subprocess.CalledProcessError as C:
         notify_error('setup', C.output)
-    return r.split()[0]
+    return r.decode().split()[0]
 
 
 def get_submodules_checkout(checkout_dir):
     os.chdir(checkout_dir)
     result = []
     try:
         submodule_status = subprocess.check_output(['git', 'submodule', 'status'])
-    except subprocess.CalledProcessError, C:
+    except subprocess.CalledProcessError as C:
         notify_error('setup', C.output)
-    for s in submodule_status.split('\n'):
+    for s in submodule_status.decode().split('\n'):
         # there is a empty line at the end...
         if s:
             result.append(s.split()[1])
@@ -145,10 +151,10 @@ def get_submodules_checkout(checkout_dir):
 
 def load_config(config_file):
     if not os.path.exists(config_file):
-        print "Error %s, do not exist" % config_file
+        print("Error %s, do not exist" % config_file)
         sys.exit(1)
     if not os.path.isfile(config_file):
-        print "Error %s is not a file" % config_file
+        print("Error %s is not a file" % config_file)
         sys.exit(1)
 
     with open(config_file) as f:
@@ -161,14 +167,14 @@ def has_submodules(checkout_dir):
     os.chdir(checkout_dir)
     try:
         r = subprocess.check_output(['git', 'submodule', 'status'])
-    except subprocess.CalledProcessError, C:
+    except subprocess.CalledProcessError as C:
         notify_error('setup', C.output)
-    return len(r) > 0
+    return len(r.decode()) > 0
 
 
 # TODO complete that
 def notify_error(stage, error):
-    print error
+    print(error)
     sys.exit(3)
 
 
@@ -189,20 +195,20 @@ def do_rsync(source):
                 '--omit-dir-times',
                 source,
                 config['remote']], stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError, C:
+    except subprocess.CalledProcessError as C:
         notify_error('setup', C.output)
-    return r
+    return r.decode()
 
 
 
 if not args.config_file:
-    print "This script take only 1 single argument, the config file"
+    print("This script take only 1 single argument, the config file")
     sys.exit(1)
 
 config = load_config(args.config_file)
 
 if 'name' not in config:
-    print "Incorrect config file: {}".format(args.config_file)
+    print("Incorrect config file: {}".format(args.config_file))
     sys.exit(1)
 
 name = config['name']
@@ -216,7 +222,7 @@ def do_rsync(source):
 
 # TODO try/except, show a better error message
 fd = os.open(lock_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
-os.write(fd, str(os.getpid()))
+os.write(fd, str(os.getpid()).encode())
 os.close(fd)
 atexit.register(os.unlink, lock_file)
 
@@ -229,7 +235,7 @@ def do_rsync(source):
 checkout_dir = os.path.expanduser("~/%s" % name)
 if not os.path.isdir(checkout_dir):
     os.unlink(lock_file)
-    print "Checkout not existing, exiting"
+    print("Checkout not existing, exiting")
     sys.exit(2)
 
 refresh_checkout(checkout_dir)
@@ -286,21 +292,21 @@ def do_rsync(source):
 if not args.no_refresh:
     try:
         result = subprocess.check_output(['git', 'stash'], stderr=subprocess.STDOUT)
-        debug_print(result)
+        debug_print(result.decode())
         result = subprocess.check_output(['git', 'stash', 'clear'], stderr=subprocess.STDOUT)
-        debug_print(result)
+        debug_print(result.decode())
         result = subprocess.check_output(['git', 'pull', '--rebase'], stderr=subprocess.STDOUT)
-        debug_print(result)
-    except subprocess.CalledProcessError, C:
+        debug_print(result.decode())
+    except subprocess.CalledProcessError as C:
         notify_error('setup', C.output)
 
 if has_submodules(checkout_dir):
     try:
         result = subprocess.check_output(['git', 'submodule', 'init'], stderr=subprocess.STDOUT)
-        debug_print(result)
+        debug_print(result.decode())
         result = subprocess.check_output(['git', 'submodule', 'sync'], stderr=subprocess.STDOUT)
-        debug_print(result)
-    except subprocess.CalledProcessError, C:
+        debug_print(result.decode())
+    except subprocess.CalledProcessError as C:
         notify_error('setup', C.output)
 
 build_subdir = builder_info[config['builder']]['build_subdir']
@@ -309,7 +315,7 @@ def do_rsync(source):
 # ensure build directory exist or creating the sync log would fail
 try:
     # TODO: use exist_ok instead of all this crap when switching to Python 3
-    os.makedirs(build_dir, mode=0775)
+    os.makedirs(build_dir, mode=0o775)
 except OSError as exc:
     if exc.errno == errno.EEXIST and os.path.isdir(build_dir):
         pass
@@ -329,15 +335,15 @@ def do_rsync(source):
             # don't use embedded libraries to build Nokogiri
             os.environ['NOKOGIRI_USE_SYSTEM_LIBRARIES'] = '1'
             result = subprocess.check_output(['bundle', 'install'], stderr=subprocess.STDOUT)
-            debug_print(result)
-        except subprocess.CalledProcessError, C:
+            debug_print(result.decode())
+        except subprocess.CalledProcessError as C:
             log_print(C.output)
             if config['remote']:
                 # copy log in build dir and sync it to make it available to users
                 shutil.copy2(log_file, sync_log_path)
                 try:
                     do_rsync(sync_log_path)
-                except subprocess.CalledProcessError, C:
+                except subprocess.CalledProcessError:
                     pass
             notify_error('install', C.output)
 
@@ -349,15 +355,15 @@ def do_rsync(source):
         command = builder_info[config['builder']]['build_command']
         syslog.syslog("Build of {}: {}".format(name, ' '.join(command)))
         result = subprocess.check_output(command, stderr=subprocess.STDOUT)
-        debug_print(result)
-    except subprocess.CalledProcessError, C:
+        debug_print(result.decode())
+    except subprocess.CalledProcessError as C:
         log_print(C.output)
         if config['remote']:
             # copy log in build dir and sync it to make it available to users
             shutil.copy2(log_file, sync_log_path)
             try:
                 do_rsync(sync_log_path)
-            except subprocess.CalledProcessError, C:
+            except subprocess.CalledProcessError:
                 pass
         notify_error('build', C.output)
 
@@ -372,11 +378,11 @@ def do_rsync(source):
         else:
             command = builder_info[config['builder']]['deploy_command']
             if command:
-                result = subprocess.check_output(command)
+                result = subprocess.check_output(command).decode()
             else:
                 result = "No deployment done: no Rsync settings provided and this builder has no reployment method defined"
         debug_print(result)
-    except subprocess.CalledProcessError, C:
+    except subprocess.CalledProcessError as C:
         notify_error('deploy', C.output)
     syslog.syslog("Build of {}: finish sync".format(name))
 else:
@@ -389,4 +395,4 @@ def do_rsync(source):
 status['submodule_commits'] = current_submodule_commits
 
 with open(status_file, 'w+') as f:
-    f.write(yaml.dump(status, default_flow_style=False))
+    f.write(yaml.safe_dump(status, default_flow_style=False))

vars/Debian.yml

@@ -27,4 +27,6 @@ package_list:
   ascii_binder: []
   planet:
     - planet-venus
+  nikola:
+    - python3-pip