Add CI to run daily from scancode develop

* Use https for git clone
* Add github actions to build licenseDB from scancode develop daily
  and push to repository

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

.github/workflows/daily-update.yml

@@ -0,0 +1,34 @@
+name: Update LicenseDB with latest updates from scancode-toolkit develop daily
+on:
+  schedule:
+    - cron: '0 16 * * 0-6'
+
+jobs:
+  update_licenseDB:
+    name: Daily LicenseDB Update
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+
+      - name: Give execute access to the script
+        run: chmod u+x ./etc/scripts/check_commit_run_update.sh
+
+      - name: Run script
+        run: ./etc/scripts/check_commit_run_update.sh
+
+      - name: Commit and Push to Publish
+        run: |
+          git config --global user.name 'LicenseDB Update GitHub Action'
+          git config --global user.email '[email protected]'
+          git commit -am "Automated LicenseDB update with latest from scancode-toolkit develop"
+          git push

Makefile

@@ -36,12 +36,12 @@ BLACK_ARGS=--exclude="docs" .
 conf:
 	@echo "-> Configure the Python venv, clone and install scancode-toolkit"
 	${PYTHON_EXE} -m venv venv
-	@git clone git@github.com:nexB/scancode-toolkit.git
+	@git clone https://github.com/nexB/scancode-toolkit.git
 	@venv/bin/pip install --upgrade pip
 	@venv/bin/pip install -e ./scancode-toolkit/
 
 restore:
-	# Restores the repository to a clean state
+	@echo "-> Restoring the repository to a clean state"
 	git clean -fd
 	rm -rf scancode-toolkit/
 	git restore --worktree docs/
@@ -54,15 +54,15 @@ clean:
 
 html:
 	@echo "-> Generate the HTML content"
-	@venv/bin/scancode --dump-license-data docs/
+	@venv/bin/scancode-license-data --path docs/
 	@echo "Available at docs/index.html"
 
 build: conf html
 
 publish:
 	@echo "-> Add changes to git"
 	@git add .
-	git commit -m "Upgrade ScanCode-toolkit to latest version"
+	git commit -m "Upgrade with latest updates from ScanCode-toolkit develop"
 	@echo "-> Push changes to main repo"
 	@git push
 

README.rst

@@ -28,6 +28,9 @@ Browse
 The web site is published at: https://scancode-licensedb.aboutcode.org/
 You can search the licenses by name, key and other attributes.
 
+This is updated daily by a GitHub action with updates from scancode-toolkit
+develop, if any.
+
 
 API
 ------
@@ -51,7 +54,9 @@ license details and texts using this license key as an identifier:
 
 
 This index and the static website is also available offline with ScanCode Toolkit as a
-command line option `--get-license-data`.
+console script available at configure/install. Sample command::
+  
+  scancode-license-data --path PATH
 
 
 Git
@@ -82,16 +87,7 @@ Build
 
 To re/generate the HTML and API content use this command::
 
-    $ make build
-
-
-Upgrade
--------
-
-To upgrade to the latest scancode-toolkit and generate the HTML and API content
-run this command::
-
-    $ make clean upgrade build publish
+    $ make restore conf html
 
 
 License

etc/scripts/check_commit.py

@@ -50,9 +50,15 @@ def cli(commit):
     data_commit_old = load_json(path=path)
     commit_old = data_commit_old["commit_hash"]
 
+    msg = (
+        f"There are no new commits in scancode-toolkit develop after {commit_old}."
+        "Aborting LicenseDB update."
+    )
+
     if commit == commit_old:
-        raise NoNewCommitException("There are no new commits in scancode-toolkit develop")
+        raise NoNewCommitException(msg)
     else:
+        click.secho(f" -> There are new commits in scancode-toolkit develop, updating last commit to {commit}.")
         data_commit_old["commit_hash"] = commit
         write_json(data=data_commit_old, path=path)
 

etc/scripts/check_commit_run_update.sh

@@ -16,7 +16,7 @@ cd ../
 # Check if scancode develop has new commits
 # if yes, update latest commit and continue
 # if no, fail here
-python3 ./etc/scripts/check_commit.py --commit "$COMMIT_LATEST"
+venv/bin/python ./etc/scripts/check_commit.py --commit "$COMMIT_LATEST"
 
 # build docs
 make html