Load and validate language codes in licenses

* Use Django's language codes mapping, extracted and vendored
* Also load an validate validate in license rules.
* Fix language codes where needed

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/licensedcode/data/licenses/cc-by-sa-2.1-jp.yml

@@ -1,5 +1,5 @@
 key: cc-by-sa-2.1-jp
-language: jp
+language: ja
 short_name: Creative Commons Attribution Share Alike 2.1 Japan
 name: Creative Commons Attribution Share Alike 2.1 Japan
 category: Copyleft Limited

src/licensedcode/data/licenses/mulanpsl-1.0.yml

@@ -1,6 +1,7 @@
 key: mulanpsl-1.0
 short_name: Mulan PSL v1
 name: Mulan Permissive Software License, Version 1
+language: zh-hans
 category: Permissive
 owner: COSCI
 homepage_url: https://license.coscl.org.cn/MulanPSL/

src/licensedcode/data/licenses/mulanpsl-2.0.yml

@@ -1,6 +1,7 @@
 key: mulanpsl-2.0
 short_name: Mulan PSL v2
 name: Mulan Permissive Software License, Version 2
+language: zh-hans
 category: Permissive
 owner: COSCI
 homepage_url: https://license.coscl.org.cn/MulanPSL2/

src/licensedcode/data/licenses/nysl-0.9982-jp.yml

@@ -1,5 +1,5 @@
 key: nysl-0.9982-jp
-language: jp
+language: ja
 short_name: NYSL 0.9982 JP
 name: NYSL 0.9982 Japanese
 category: Permissive

src/licensedcode/data/rules/mit_zh.yml

@@ -1,3 +1,3 @@
 license_expression: mit
-language: zh
+language: zh-hans
 is_license_notice: yes

src/licensedcode/languages.py.ABOUT

@@ -1,8 +1,13 @@
 about_resource: languages.py
+type: github
+name: django
+namespace: django
 version: ce586ed6
-download_url: https://raw.githubusercontent.com/django/django/ce586ed6931092d3a5f06df9031cdeb891793ddb/django/conf/locale/__init__.py
+subpath: conf/locale/__init__.py
 package_url: pkg:github/django/django@ce586ed6931092d3a5f06df9031cdeb891793ddb#conf/locale/__init__.py
 
+download_url: https://raw.githubusercontent.com/django/django/ce586ed6931092d3a5f06df9031cdeb891793ddb/django/conf/locale/__init__.py
+
 description: List of ISO languages copied from Django
 homepage_url: https://github.com/django/django
 copyright: Copyright (c) Django Software Foundation and individual contributors.

src/licensedcode/models.py

@@ -33,6 +33,7 @@
 from licensedcode import MIN_MATCH_HIGH_LENGTH
 from licensedcode import MIN_MATCH_LENGTH
 from licensedcode import SMALL_RULE
+from licensedcode.languages import LANG_INFO as known_languages
 from licensedcode.spans import Span
 from licensedcode.tokenize import index_tokenizer
 from licensedcode.tokenize import index_tokenizer_with_stopwords
@@ -535,7 +536,7 @@ def validate(licenses, verbose=False, no_dupe_urls=False):
                 by_short_name_lowered[lic.short_name].append(lic)
 
             if lic.key != lic.key.lower():
-                error('Incorrect license key case. Should be lowercase.')
+                error('Incorrect license key case: must be all lowercase.')
 
             if len(lic.key) > 50:
                 error('key must be 50 characters or less.')
@@ -560,13 +561,19 @@ def validate(licenses, verbose=False, no_dupe_urls=False):
             if not lic.owner:
                 error('No owner: Use "Unspecified" if not known.')
 
+            lang = lic.language
+            if lang and lang != 'en' and lang not in known_languages:
+                error(f'Unknown language: {lang}')
+
             if lic.is_unknown:
                 if not 'unknown' in lic.key:
-                    error('is_unknown can be true only for licenses with '
-                          '"unknown " in their key string.')
+                    error(
+                        'is_unknown can be true only for licenses with '
+                        '"unknown " in their key string.'
+                    )
 
             if lic.is_generic and lic.is_unknown:
-                error('is_generic and is_unknown are incompatible')
+                error('is_generic and is_unknown flags are incompatible')
 
             # URLS dedupe and consistency
             if no_dupe_urls:
@@ -1474,10 +1481,14 @@ def validate(self, licensing=None):
             if any(ignorables):
                 yield 'is_false_positive rule cannot have ignorable_* attributes.'
 
-        if not (0 <= self.minimum_coverage <= 100):
-            yield 'Invalid rule minimum_coverage. Should be between 0 and 100.'
+        lang = self.language
+        if lang and lang != 'en' and lang not in known_languages:
+            yield f'Unknown language: {lang}'
 
         if not is_false_positive:
+            if not (0 <= self.minimum_coverage <= 100):
+                yield 'Invalid rule minimum_coverage. Should be between 0 and 100.'
+
             if not (0 <= self.relevance <= 100):
                 yield 'Invalid rule relevance. Should be between 0 and 100.'
 
@@ -1877,6 +1888,8 @@ def load(self):
         self.ignorable_urls = data.get('ignorable_urls', [])
         self.ignorable_emails = data.get('ignorable_emails', [])
 
+        self.language = data.get('language') or 'en'
+
         return self
 
     def set_relevance(self):

tests/licensedcode/data/models/rule_validate_lang/0050_oclc-2.0.RULE

@@ -0,0 +1 @@
+http://www.oclc.org/research/activities/software/license/v2final.htm

tests/licensedcode/data/models/rule_validate_lang/0050_oclc-2.0.yml

@@ -0,0 +1,3 @@
+license_expression: oclc-2.0 AND baz
+language: foobar
+notes: spdx_source_url

tests/licensedcode/data/models/rule_validate_lang/0051_gpl-2.0.RULE

@@ -0,0 +1 @@
+   This program is free software; you can redistribute it and/or modify