fixes #485

Author: jph00

CHANGELOG.md

@@ -2,6 +2,62 @@
 
 <!-- do not remove -->
 
+## 1.5.26
+
+
+### Bugs Squashed
+
+- UnicodeDecodeError occurs when reading `settings.ini` file containing CJK characters on Windows, due to missing encoding parameter ([#483](https://github.com/fastai/fastcore/issues/483))
+  - When I tried to preview/test/prepare an nbdev project with `settings.ini` (UTF-8 encorded) which containing some CKJ (or maybe other non-ascii) characters, an error such as `UnicodeDecodeError: 'cp932' codec can't decode byte 0x82 in position 725: illegal multibyte sequence` ocurred.
+
+<details><summary>Example of settings and full error message</summary>
+
+When a setting file containing a line like
+```
+description = サンプル プロジェクト (sample project)
+```
+
+and nbdev_* command executed, output is like below:
+```
+$ nbdev_preview.exe
+Traceback (most recent call last):
+  File "C:\Users\<user_home>\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "C:\Users\<user_home>\AppData\Local\Programs\Python\Python310\lib\runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "<path_to_venv>\Scripts\nbdev_preview.exe\__main__.py", line 7, in <module>
+  File "<path_to_venv>\lib\site-packages\fastcore\script.py", line 119, in _f
+    return tfunc(**merge(args, args_from_prog(func, xtra)))
+  File "<path_to_venv>\lib\site-packages\nbdev\quarto.py", line 278, in preview
+    nbdev_quarto.__wrapped__(path, preview=True, **kwargs)
+  File "<path_to_venv>\lib\site-packages\nbdev\quarto.py", line 256, in nbdev_quarto
+    nbdev.doclinks._build_modidx(skip_exists=True)
+  File "<path_to_venv>\lib\site-packages\nbdev\doclinks.py", line 74, in _build_modidx
+    if dest is None: dest = get_config().lib_path
+  File "<path_to_venv>\lib\site-packages\nbdev\config.py", line 199, in get_config
+    cfg = Config(cfg_file.parent, cfg_file.name, extra_files=extra_files, types=_types)
+  File "<path_to_venv>\lib\site-packages\fastcore\foundation.py", line 258, in __init__
+    found = [Path(o) for o in self._cfg.read(L(extra_files)+[self.config_file])]#, encoding='utf-8')]
+  File "C:\Users\<user_home>\AppData\Local\Programs\Python\Python310\lib\configparser.py", line 698, in read
+    self._read(fp, filename)
+  File "C:\Users\<user_home>\AppData\Local\Programs\Python\Python310\lib\configparser.py", line 1021, in _read
+    for lineno, line in enumerate(fp, start=1):
+UnicodeDecodeError: 'cp932' codec can't decode byte 0x82 in position 725: illegal multibyte sequence
+```
+
+Version info:
+Operating system: Windows 11 Pro (Japanese)
+Python 3.10.6
+nbdev 2.1.7
+
+</details>
+
+This error is likely caused due to no encoding being specified here:
+https://github.com/fastai/fastcore/blob/894bf94a3fcab91c85f05bc9a974a747533e9040/fastcore/foundation.py#L258
+
+The error seems to be resolved by adding `encoding='utf-8'` to the argument of the ConfigParser.read() method.
+
+
 ## 1.5.25
 
 ### New Features

fastcore/__init__.py

@@ -1 +1 @@
-__version__ = "1.5.25"
+__version__ = "1.5.26"

fastcore/docments.py

@@ -34,22 +34,22 @@ def parse_docstring(sym):
     docs = docstring(sym)
     return AttrDict(**docscrape.NumpyDocString(docstring(sym)))
 
-# %% ../nbs/06_docments.ipynb 17
+# %% ../nbs/06_docments.ipynb 16
 def isdataclass(s):
     "Check if `s` is a dataclass but not a dataclass' instance"
     return is_dataclass(s) and isclass(s)
 
-# %% ../nbs/06_docments.ipynb 18
+# %% ../nbs/06_docments.ipynb 17
 def get_dataclass_source(s):
     "Get source code for dataclass `s`"
     return getsource(s) if not getattr(s, "__module__") == '__main__' else ""
 
-# %% ../nbs/06_docments.ipynb 19
+# %% ../nbs/06_docments.ipynb 18
 def get_source(s):
     "Get source code for string, function object or dataclass `s`"
     return getsource(s) if isfunction(s) or ismethod(s) else get_dataclass_source(s) if isdataclass(s) else s
 
-# %% ../nbs/06_docments.ipynb 20
+# %% ../nbs/06_docments.ipynb 19
 def _parses(s):
     "Parse Python code in string, function object or dataclass `s`"
     return parse(dedent(get_source(s)))
@@ -78,10 +78,10 @@ def _param_locs(s, returns=True):
             return res
     return None
 
-# %% ../nbs/06_docments.ipynb 21
+# %% ../nbs/06_docments.ipynb 20
 empty = Parameter.empty
 
-# %% ../nbs/06_docments.ipynb 22
+# %% ../nbs/06_docments.ipynb 21
 def _get_comment(line, arg, comments, parms):
     if line in comments: return comments[line].strip()
     line -= 1
@@ -95,7 +95,7 @@ def _get_full(anno, name, default, docs):
     if anno==empty and default!=empty: anno = type(default)
     return AttrDict(docment=docs.get(name), anno=anno, default=default)
 
-# %% ../nbs/06_docments.ipynb 23
+# %% ../nbs/06_docments.ipynb 22
 def _merge_doc(dm, npdoc):
     if not npdoc: return dm
     if not dm.anno or dm.anno==empty: dm.anno = npdoc.type
@@ -108,14 +108,14 @@ def _merge_docs(dms, npdocs):
     if 'return' in dms: params['return'] = _merge_doc(dms['return'], npdocs['Returns'])
     return params
 
-# %% ../nbs/06_docments.ipynb 24
+# %% ../nbs/06_docments.ipynb 23
 def _get_property_name(p):
     "Get the name of property `p`"
     if hasattr(p, 'fget'):
         return p.fget.func.__qualname__ if hasattr(p.fget, 'func') else p.fget.__qualname__
     else: return next(iter(re.findall(r'\'(.*)\'', str(p)))).split('.')[-1]
 
-# %% ../nbs/06_docments.ipynb 25
+# %% ../nbs/06_docments.ipynb 24
 def get_name(obj):
     "Get the name of `obj`"
     if hasattr(obj, '__name__'):       return obj.__name__
@@ -124,7 +124,7 @@ def get_name(obj):
     elif type(obj)==property:          return _get_property_name(obj)
     else:                              return str(obj).split('.')[-1]
 
-# %% ../nbs/06_docments.ipynb 27
+# %% ../nbs/06_docments.ipynb 26
 def qual_name(obj):
     "Get the qualified name of `obj`"
     if hasattr(obj,'__qualname__'): return obj.__qualname__
@@ -161,8 +161,8 @@ def docments(elt, full=False, **kwargs):
 
     def _update_docments(f, r):
         if hasattr(f, '__delwrap__'): _update_docments(f.__delwrap__, r)
-        r.update({k:v for k,v in _docments(f, **kwargs).items()
-                  if k in params and (full or v.get('docment',None))})
+        r.update({k:v for k,v in _docments(f, **kwargs).items() if k in params
+                  and (v.get('docment', None) or not nested_idx(r, k, 'docment'))})
 
     _update_docments(elt, r)
     if not full: r = {k:v['docment'] for k,v in r.items()}

nbs/06_docments.ipynb

@@ -205,13 +205,6 @@
     "parse_docstring(add_np)"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Usage"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -398,6 +391,13 @@
     "assert qual_name(docscrape) == 'fastcore.docscrape'"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Docments"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -441,8 +441,8 @@
     "\n",
     "    def _update_docments(f, r):\n",
     "        if hasattr(f, '__delwrap__'): _update_docments(f.__delwrap__, r)\n",
-    "        r.update({k:v for k,v in _docments(f, **kwargs).items()\n",
-    "                  if k in params and (full or v.get('docment',None))})\n",
+    "        r.update({k:v for k,v in _docments(f, **kwargs).items() if k in params\n",
+    "                  and (v.get('docment', None) or not nested_idx(r, k, 'docment'))})\n",
     "\n",
     "    _update_docments(elt, r)\n",
     "    if not full: r = {k:v['docment'] for k,v in r.items()}\n",
@@ -465,17 +465,15 @@
      "data": {
       "text/markdown": [
        "```json\n",
-       "{ 'a': 'The first operand',\n",
-       "  'b': 'This is the second of the operands to the *addition* operator.\\n'\n",
-       "       'Note that passing a negative value here is the equivalent of the '\n",
-       "       '*subtraction* operator.',\n",
-       "  'return': \"The result is calculated using Python's builtin `+` operator.\"}\n",
+       "{ 'a': 'the 1st number to add',\n",
+       "  'b': 'the 2nd number to add',\n",
+       "  'return': 'the result of adding `a` to `b`'}\n",
        "```"
       ],
       "text/plain": [
-       "{'a': 'The first operand',\n",
-       " 'b': 'This is the second of the operands to the *addition* operator.\\nNote that passing a negative value here is the equivalent of the *subtraction* operator.',\n",
-       " 'return': \"The result is calculated using Python's builtin `+` operator.\"}"
+       "{'a': 'the 1st number to add',\n",
+       " 'b': 'the 2nd number to add',\n",
+       " 'return': 'the result of adding `a` to `b`'}"
       ]
      },
      "execution_count": null,
@@ -484,6 +482,13 @@
     }
    ],
    "source": [
+    "def add(\n",
+    "    a:int, # the 1st number to add\n",
+    "    b=0,   # the 2nd number to add\n",
+    ")->int:    # the result of adding `a` to `b`\n",
+    "    \"The sum of two numbers.\"\n",
+    "    return a+b\n",
+    "\n",
     "docments(add)"
    ]
   },
@@ -505,27 +510,21 @@
        "```json\n",
        "{ 'a': { 'anno': 'int',\n",
        "         'default': <class 'inspect._empty'>,\n",
-       "         'docment': 'The first operand'},\n",
-       "  'b': { 'anno': 'int',\n",
-       "         'default': <class 'inspect._empty'>,\n",
-       "         'docment': 'This is the second of the operands to the *addition* '\n",
-       "                    'operator.\\n'\n",
-       "                    'Note that passing a negative value here is the equivalent '\n",
-       "                    'of the *subtraction* operator.'},\n",
+       "         'docment': 'the 1st number to add'},\n",
+       "  'b': { 'anno': <class 'int'>,\n",
+       "         'default': 0,\n",
+       "         'docment': 'the 2nd number to add'},\n",
        "  'return': { 'anno': 'int',\n",
        "              'default': <class 'inspect._empty'>,\n",
-       "              'docment': \"The result is calculated using Python's builtin `+` \"\n",
-       "                         'operator.'}}\n",
+       "              'docment': 'the result of adding `a` to `b`'}}\n",
        "```"
       ],
       "text/plain": [
-       "{'a': {'docment': 'The first operand',\n",
-       "  'anno': 'int',\n",
-       "  'default': inspect._empty},\n",
-       " 'b': {'docment': 'This is the second of the operands to the *addition* operator.\\nNote that passing a negative value here is the equivalent of the *subtraction* operator.',\n",
+       "{'a': {'docment': 'the 1st number to add',\n",
        "  'anno': 'int',\n",
        "  'default': inspect._empty},\n",
-       " 'return': {'docment': \"The result is calculated using Python's builtin `+` operator.\",\n",
+       " 'b': {'docment': 'the 2nd number to add', 'anno': int, 'default': 0},\n",
+       " 'return': {'docment': 'the result of adding `a` to `b`',\n",
        "  'anno': 'int',\n",
        "  'default': inspect._empty}}"
       ]
@@ -557,11 +556,11 @@
        "```json\n",
        "{ 'anno': <class 'int'>,\n",
        "  'default': <class 'inspect._empty'>,\n",
-       "  'docment': 'The first operand'}\n",
+       "  'docment': 'the 1st number to add'}\n",
        "```"
       ],
       "text/plain": [
-       "{'docment': 'The first operand', 'anno': int, 'default': inspect._empty}"
+       "{'docment': 'the 1st number to add', 'anno': int, 'default': inspect._empty}"
       ]
      },
      "execution_count": null,
@@ -663,11 +662,11 @@
      "data": {
       "text/markdown": [
        "```json\n",
-       "{'a': 'First operand', 'b': '2nd operand'}\n",
+       "{'a': 'First operand', 'b': '2nd operand', 'return': None}\n",
        "```"
       ],
       "text/plain": [
-       "{'a': 'First operand', 'b': '2nd operand'}"
+       "{'a': 'First operand', 'b': '2nd operand', 'return': None}"
       ]
      },
      "execution_count": null,
@@ -688,11 +687,11 @@
      "data": {
       "text/markdown": [
        "```json\n",
-       "{'return': 'Integral result of addition operator'}\n",
+       "{'return': 'Integral result of addition operator', 'self': None}\n",
        "```"
       ],
       "text/plain": [
-       "{'return': 'Integral result of addition operator'}"
+       "{'self': None, 'return': 'Integral result of addition operator'}"
       ]
      },
      "execution_count": null,
@@ -855,7 +854,7 @@
     "                     foo:str, # docment for parameter foo\n",
     "                     ):...\n",
     "    \n",
-    "test_eq(docments(_F.class_method), {'foo': 'docment for parameter foo'})"
+    "test_eq(docments(_F.class_method), {'foo': 'docment for parameter foo', 'return': None})"
    ]
   },
   {
@@ -874,18 +873,6 @@
     "from fastcore.meta import delegates"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def _a(a:int=2): return a # First\n",
-    "\n",
-    "@delegates(_a)\n",
-    "def _b(b:str, **kwargs): return b, (_a(**kwargs)) # Second"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -895,11 +882,11 @@
      "data": {
       "text/markdown": [
        "```json\n",
-       "{'a': 'First', 'b': 'Second'}\n",
+       "{'a': 'First', 'b': 'Second', 'return': None}\n",
        "```"
       ],
       "text/plain": [
-       "{'a': 'First', 'b': 'Second'}"
+       "{'a': 'First', 'return': None, 'b': 'Second'}"
       ]
      },
      "execution_count": null,
@@ -908,6 +895,11 @@
     }
    ],
    "source": [
+    "def _a(a:int=2): return a # First\n",
+    "\n",
+    "@delegates(_a)\n",
+    "def _b(b:str, **kwargs): return b, (_a(**kwargs)) # Second\n",
+    "\n",
     "docments(_b)"
    ]
   },
@@ -923,7 +915,9 @@
     "\n",
     "@delegates(_c)\n",
     "def _d(c:int, # First\n",
-    "       b:str, **kwargs): return c, _c(b, **kwargs)"
+    "       b:str, **kwargs\n",
+    "      )->int:\n",
+    "    return c, _c(b, **kwargs)"
    ]
   },
   {
@@ -934,7 +928,7 @@
    "source": [
     "#|hide\n",
     "test_eq(docments(_c, full=True)['b']['docment'],'Second')\n",
-    "test_eq(docments(_d, full=True)['b']['docment'],None)\n",
+    "test_eq(docments(_d, full=True)['b']['docment'],'Second')\n",
     "_argset = {'a', 'b', 'c', 'return'}\n",
     "test_eq(docments(_d, full=True).keys() & _argset, _argset) # _d has the args a,b,c and return"
    ]

settings.ini

@@ -7,7 +7,7 @@ author = Jeremy Howard and Sylvain Gugger
 author_email = [email protected]
 copyright = fast.ai
 branch = master
-version = 1.5.25
+version = 1.5.26
 min_python = 3.7
 audience = Developers
 language = English