fixes #336

Author: jph00

fastcore/_nbdev.py

@@ -163,6 +163,7 @@
          "Path.mk_write": "03_xtras.ipynb",
          "Path.ls": "03_xtras.ipynb",
          "Path.__repr__": "03_xtras.ipynb",
+         "Path.delete": "03_xtras.ipynb",
          "truncstr": "03_xtras.ipynb",
          "spark_chars": "03_xtras.ipynb",
          "sparkline": "03_xtras.ipynb",
@@ -201,6 +202,7 @@
          "urlcheck": "03b_net.ipynb",
          "urlclean": "03b_net.ipynb",
          "urlretrieve": "03b_net.ipynb",
+         "urldest": "03b_net.ipynb",
          "urlsave": "03b_net.ipynb",
          "urlvalid": "03b_net.ipynb",
          "urlrequest": "03b_net.ipynb",

fastcore/foundation.py

@@ -246,16 +246,22 @@ def read_config_file(file, **kwargs):
 # Cell
 class Config:
     "Reading and writing `ConfigParser` ini files"
-    def __init__(self, cfg_path, cfg_name):
+    def __init__(self, cfg_path, cfg_name, create=None):
         cfg_path = Path(cfg_path).expanduser().absolute()
         self.config_path,self.config_file = cfg_path,cfg_path/cfg_name
-        assert self.config_file.exists(), f"Could not find {cfg_name}"
+        if not self.config_file.exists():
+            if create:
+                self.d = create
+                cfg_path.mkdir(exist_ok=True, parents=True)
+                self.save()
+            else: raise Exception(f"Could not find {cfg_name}")
         self.d = read_config_file(self.config_file)
 
     def __setitem__(self,k,v): self.d[k] = str(v)
     def __contains__(self,k):  return k in self.d
     def save(self):            save_config_file(self.config_file,self.d)
     def __getattr__(self,k):   return stop(AttributeError(k)) if k=='d' or k not in self.d else self.get(k)
+    def __getitem__(self,k):   return stop(IndexError(k)) if k not in self.d else self.get(k)
     def get(self,k,default=None): return self.d.get(k, default)
 
     def path(self,k,default=None):

fastcore/net.py

@@ -10,8 +10,8 @@
            'HTTP422UnprocessableEntityError', 'HTTP423LockedError', 'HTTP424FailedDependencyError',
            'HTTP425TooEarlyError', 'HTTP426UpgradeRequiredError', 'HTTP428PreconditionRequiredError',
            'HTTP429TooManyRequestsError', 'HTTP431HeaderFieldsTooLargeError', 'HTTP451LegalReasonsError', 'urlopen',
-           'urlread', 'urljson', 'urlcheck', 'urlclean', 'urlretrieve', 'urlsave', 'urlvalid', 'urlrequest', 'urlsend',
-           'do_request', 'start_server', 'start_client']
+           'urlread', 'urljson', 'urlcheck', 'urlclean', 'urlretrieve', 'urldest', 'urlsave', 'urlvalid', 'urlrequest',
+           'urlsend', 'do_request', 'start_server', 'start_client']
 
 # Cell
 from .imports import *
@@ -164,12 +164,16 @@ def urlretrieve(url, filename=None, reporthook=None, data=None):
     return filename,headers
 
 # Cell
-def urlsave(url, dest=None, reporthook=None):
-    "Retrieve `url` and save based on its name"
+def urldest(url, dest=None):
     name = urlclean(Path(url).name)
     if dest is None: dest = name
     dest = Path(dest)
-    if dest.is_dir(): dest = dest/name
+    return dest/name if dest.is_dir() else dest
+
+# Cell
+def urlsave(url, dest=None, reporthook=None):
+    "Retrieve `url` and save based on its name"
+    dest = urldest(url, dest)
     dest.parent.mkdir(parents=True, exist_ok=True)
     nm,msg = urlretrieve(url, dest, reporthook)
     return nm

fastcore/xtras.py

@@ -13,7 +13,7 @@
 from functools import wraps
 
 import mimetypes,pickle,random,json,subprocess,shlex,bz2,gzip,zipfile,tarfile
-import imghdr,struct,distutils.util,tempfile,time,string,collections
+import imghdr,struct,distutils.util,tempfile,time,string,collections,shutil
 from contextlib import contextmanager,ExitStack
 from pdb import set_trace
 from datetime import datetime, timezone
@@ -148,15 +148,18 @@ def loads_multi(s:str):
         s = s[pos:]
 
 # Cell
-def untar_dir(fname, dest):
-    "untar `file` into `dest`"
+def untar_dir(fname, dest, rename=False, overwrite=False):
+    "untar `file` into `dest`, creating a directory if the root contains more than one item"
     with tempfile.TemporaryDirectory(dir='.') as d:
-        out = Path(d)/'out'
+        out = Path(d)/remove_suffix(Path(fname).stem, '.tar')
         out.mkdir()
         shutil.unpack_archive(fname, out)
         ls = out.ls()
-        if len(ls) == 1: shutil.move(str(ls[0]), dest)
-        else: shutil.move(str(out), dest/remove_suffix(Path(fname).stem, '.tar'))
+        src = ls[0] if len(ls) == 1 else out
+        dest = dest/(out if rename else src).name
+        if overwrite: shutil.rmtree(dest, ignore_errors=True)
+        shutil.move(str(src), dest)
+        return dest
 
 # Cell
 def repo_details(url):
@@ -247,6 +250,14 @@ def __repr__(self:Path):
         except: pass
     return f"Path({self.as_posix()!r})"
 
+# Cell
+@patch
+def delete(self:Path):
+    "Delete a file, symlink, or directory tree"
+    if not self.exists(): return
+    if self.is_dir(): shutil.rmtree(self)
+    else: self.unlink()
+
 # Cell
 def truncstr(s:str, maxlen:int, suf:str='…', space='')->str:
     "Truncate `s` to length `maxlen`, adding suffix `suf` if truncated"

nbs/02_foundation.ipynb

@@ -792,7 +792,7 @@
     {
      "data": {
       "text/plain": [
-       "[7, 0, 'j']"
+       "[2, 'k', 11]"
       ]
      },
      "execution_count": null,
@@ -1880,16 +1880,22 @@
     "#export\n",
     "class Config:\n",
     "    \"Reading and writing `ConfigParser` ini files\"\n",
-    "    def __init__(self, cfg_path, cfg_name):\n",
+    "    def __init__(self, cfg_path, cfg_name, create=None):\n",
     "        cfg_path = Path(cfg_path).expanduser().absolute()\n",
     "        self.config_path,self.config_file = cfg_path,cfg_path/cfg_name\n",
-    "        assert self.config_file.exists(), f\"Could not find {cfg_name}\"\n",
+    "        if not self.config_file.exists():\n",
+    "            if create:\n",
+    "                self.d = create\n",
+    "                cfg_path.mkdir(exist_ok=True, parents=True)\n",
+    "                self.save()\n",
+    "            else: raise Exception(f\"Could not find {cfg_name}\")\n",
     "        self.d = read_config_file(self.config_file)\n",
     "\n",
     "    def __setitem__(self,k,v): self.d[k] = str(v)\n",
     "    def __contains__(self,k):  return k in self.d\n",
     "    def save(self):            save_config_file(self.config_file,self.d)\n",
     "    def __getattr__(self,k):   return stop(AttributeError(k)) if k=='d' or k not in self.d else self.get(k)\n",
+    "    def __getitem__(self,k):   return stop(IndexError(k)) if k not in self.d else self.get(k)\n",
     "    def get(self,k,default=None): return self.d.get(k, default)\n",
     "\n",
     "    def path(self,k,default=None):\n",
@@ -1910,13 +1916,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "try:\n",
-    "    save_config_file('../tmp.ini', _d)\n",
-    "    cfg = Config('..', 'tmp.ini')\n",
+    "try: cfg = Config('..', 'tmp.ini', create=_d)\n",
     "finally: os.unlink('../tmp.ini')\n",
     "\n",
     "test_eq(cfg.user,'fastai')\n",
-    "test_eq(cfg.get('some_path'), 'test')\n",
+    "test_eq(cfg['some_path'], 'test')\n",
     "test_eq(cfg.path('some_path'), Path('../test').absolute())\n",
     "test_eq(cfg.get('foo','bar'),'bar')"
    ]

nbs/03_xtras.ipynb

@@ -22,7 +22,7 @@
     "from functools import wraps\n",
     "\n",
     "import mimetypes,pickle,random,json,subprocess,shlex,bz2,gzip,zipfile,tarfile\n",
-    "import imghdr,struct,distutils.util,tempfile,time,string,collections\n",
+    "import imghdr,struct,distutils.util,tempfile,time,string,collections,shutil\n",
     "from contextlib import contextmanager,ExitStack\n",
     "from pdb import set_trace\n",
     "from datetime import datetime, timezone\n",
@@ -618,7 +618,7 @@
     {
      "data": {
       "text/plain": [
-       "['c', 'b', 'e', 'f', 'g', 'a', 'd', 'h']"
+       "['g', 'b', 'c', 'e', 'a', 'h', 'd', 'f']"
       ]
      },
      "execution_count": null,
@@ -1007,22 +1007,18 @@
    "outputs": [],
    "source": [
     "#export\n",
-    "def untar_dir(fname, dest):\n",
-    "    \"untar `file` into `dest`\"\n",
+    "def untar_dir(fname, dest, rename=False, overwrite=False):\n",
+    "    \"untar `file` into `dest`, creating a directory if the root contains more than one item\"\n",
     "    with tempfile.TemporaryDirectory(dir='.') as d:\n",
-    "        out = Path(d)/'out'\n",
+    "        out = Path(d)/remove_suffix(Path(fname).stem, '.tar')\n",
     "        out.mkdir()\n",
     "        shutil.unpack_archive(fname, out)\n",
     "        ls = out.ls()\n",
-    "        if len(ls) == 1: shutil.move(str(ls[0]), dest)\n",
-    "        else: shutil.move(str(out), dest/remove_suffix(Path(fname).stem, '.tar'))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "If the contents of `fname` contain just one file or directory, it is placed directly in `dest`:"
+    "        src = ls[0] if len(ls) == 1 else out\n",
+    "        dest = dest/(out if rename else src).name\n",
+    "        if overwrite: shutil.rmtree(dest, ignore_errors=True)\n",
+    "        shutil.move(str(src), dest)\n",
+    "        return dest"
    ]
   },
   {
@@ -1041,6 +1037,13 @@
     "            test_eq(d2.ls(), [d2/foldername])"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If the contents of `fname` contain just one file or directory, it is placed directly in `dest`:"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -1134,22 +1137,10 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "popenargs (['cmd', '/c', 'echo'],)\n",
-      "kwargs {'stdout': -1, 'stderr': -1}\n",
-      "popenargs (('cmd', '/c', 'pip', '--version'),)\n",
-      "kwargs {'stdout': -1, 'stderr': -1}\n",
-      "popenargs (['cmd', '/c', 'pip', '--version'],)\n",
-      "kwargs {'stdout': -1, 'stderr': -1}\n"
-     ]
-    },
     {
      "data": {
       "text/plain": [
-       "'pip 21.0.1 from D:\\\\programs\\\\envs\\\\nbdev\\\\lib\\\\site-packages\\\\pip (python 3.8)'"
+       "'pip 21.1.3 from /home/jhoward/miniconda3/lib/python3.8/site-packages/pip (python 3.8)'"
       ]
      },
      "execution_count": null,
@@ -1167,20 +1158,7 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "popenargs (['cmd', '/c', 'dir', '/p'],)\n",
-      "kwargs {'stdout': -1, 'stderr': -1}\n",
-      "popenargs (['cmd', '/c', 'dir', '/p'],)\n",
-      "kwargs {'stdout': -1, 'stderr': -1}\n",
-      "popenargs (('cmd', '/c', 'dir', '/p'),)\n",
-      "kwargs {'stdout': -1, 'stderr': -1}\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "if sys.platform == 'win32':\n",
     "    assert 'ipynb' in run('cmd /c dir /p')\n",
@@ -1203,16 +1181,7 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "popenargs (['cmd', '/c', 'findstr', 'asdfds', '00_test.ipynb'],)\n",
-      "kwargs {'stdout': -1, 'stderr': -1}\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "if sys.platform == 'win32':\n",
     "    test_eq(run('cmd /c findstr asdfds 00_test.ipynb', ignore_ex=True)[0], 1)\n",
@@ -1231,16 +1200,7 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "popenargs (['cmd', '/c', 'echo', 'hi'],)\n",
-      "kwargs {'stdout': -1, 'stderr': -1}\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "if sys.platform == 'win32':\n",
     "    # why I ignore as_types, because every time nbdev_clean_nbs will update \\n to \\r\\n\n",
@@ -1394,7 +1354,7 @@
     {
      "data": {
       "text/plain": [
-       "Path('.gitattributes')"
+       "Path('05_transform.ipynb')"
       ]
      },
      "execution_count": null,
@@ -1428,7 +1388,7 @@
     {
      "data": {
       "text/plain": [
-       "(Path('../fastcore/all.py'), Path('00_test.ipynb'))"
+       "(Path('../fastcore/all.py'), Path('05_transform.ipynb'))"
       ]
      },
      "execution_count": null,
@@ -1494,6 +1454,21 @@
     "finally: Path.BASE_PATH = None"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#export\n",
+    "@patch\n",
+    "def delete(self:Path):\n",
+    "    \"Delete a file, symlink, or directory tree\"\n",
+    "    if not self.exists(): return\n",
+    "    if self.is_dir(): shutil.rmtree(self)\n",
+    "    else: self.unlink()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

nbs/03b_net.ipynb

@@ -401,12 +401,23 @@
    "outputs": [],
    "source": [
     "#export\n",
-    "def urlsave(url, dest=None, reporthook=None):\n",
-    "    \"Retrieve `url` and save based on its name\"\n",
+    "def urldest(url, dest=None):\n",
     "    name = urlclean(Path(url).name)\n",
     "    if dest is None: dest = name\n",
     "    dest = Path(dest)\n",
-    "    if dest.is_dir(): dest = dest/name\n",
+    "    return dest/name if dest.is_dir() else dest"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#export\n",
+    "def urlsave(url, dest=None, reporthook=None):\n",
+    "    \"Retrieve `url` and save based on its name\"\n",
+    "    dest = urldest(url, dest)\n",
     "    dest.parent.mkdir(parents=True, exist_ok=True)\n",
     "    nm,msg = urlretrieve(url, dest, reporthook)\n",
     "    return nm"
@@ -672,6 +683,7 @@
       "Converted 05_transform.ipynb.\n",
       "Converted 07_meta.ipynb.\n",
       "Converted 08_script.ipynb.\n",
+      "Converted index.ipynb.\n",
       "Converted parallel_win.ipynb.\n"
      ]
     }