From 394c5f0a29cb09a48fc9014f500a9592f6c0c710 Mon Sep 17 00:00:00 2001 From: Didier Vezinet Date: Wed, 29 Jan 2025 15:58:52 -0500 Subject: [PATCH 1/5] [#181] overwrite implemented --- datastock/_class0.py | 4 +++- datastock/_saveload.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/datastock/_class0.py b/datastock/_class0.py index 8cfc15e..3596310 100644 --- a/datastock/_class0.py +++ b/datastock/_class0.py @@ -164,8 +164,9 @@ def save( path=None, name=None, sep=None, - verb=True, + overwrite=None, return_pfe=False, + verb=True, ): lsep = [';', '&', '?', '#', ',', '~', '.', '-', '_'] @@ -194,6 +195,7 @@ def save( sep=sep, path=path, name=name, + overwrite=overwrite, clsname=self.__class__.__name__, return_pfe=return_pfe, verb=verb, diff --git a/datastock/_saveload.py b/datastock/_saveload.py index 187991f..6933668 100644 --- a/datastock/_saveload.py +++ b/datastock/_saveload.py @@ -30,6 +30,7 @@ def save( name=None, path=None, clsname=None, + overwrite=None, return_pfe=None, verb=None, ): @@ -37,6 +38,7 @@ def save( # ------------ # check inputs + # ------------ # path path = _generic_check._check_var( @@ -63,6 +65,13 @@ def save( types=str, ) + # overwrite + overwrite = _generic_check._check_var( + overwrite, 'overwrite', + default=False, + types=bool, + ) + # verb verb = _generic_check._check_var( verb, 'verb', @@ -79,6 +88,7 @@ def save( # ---------------------- # save / print / return + # ---------------------- user = getpass.getuser() dt = dtm.datetime.now().strftime("%Y%m%d-%H%M%S") @@ -87,8 +97,26 @@ def save( # add sep dflat[_KEY_SEP] = sep - # save + # pfe pfe = os.path.join(path, name) + + # check vs existing + if os.path.isfile(pfe): + if overwrite is True: + msg = ( + "Overwriting existing file:\n" + f"\t{pfe}" + ) + warnings.warn(msg) + else: + msg = ( + "File already existing!\n" + "\t=> use overwrite = True to overwrite\n" + f"\t{pfe}" + ) + raise Exception(msg) + + # save np.savez(pfe, **dflat) # print From ae7aae9e07c581a77edfb4750a7214b07830bfb3 Mon Sep 17 00:00:00 2001 From: Didier Vezinet Date: Wed, 29 Jan 2025 15:59:23 -0500 Subject: [PATCH 2/5] [#181] Additional spaces implemented --- datastock/_saveload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastock/_saveload.py b/datastock/_saveload.py index 6933668..223cdd5 100644 --- a/datastock/_saveload.py +++ b/datastock/_saveload.py @@ -121,7 +121,7 @@ def save( # print if verb: - msg = f"Saved in:\n\t{pfe}" + msg = f"\nSaved in:\n\t{pfe}\n" print(msg) # return From d8f13671a3607c3e674b558261ad9d9cb01ec3d9 Mon Sep 17 00:00:00 2001 From: Didier Vezinet Date: Wed, 29 Jan 2025 16:23:08 -0500 Subject: [PATCH 3/5] [#181] pfe implemented --- datastock/_class0.py | 2 + datastock/_saveload.py | 119 +++++++++++++++++++++++++++++++---------- 2 files changed, 92 insertions(+), 29 deletions(-) diff --git a/datastock/_class0.py b/datastock/_class0.py index 3596310..26b909d 100644 --- a/datastock/_class0.py +++ b/datastock/_class0.py @@ -161,6 +161,7 @@ def __hash__(self, *args, **kargs): def save( self, + pfe=None, path=None, name=None, sep=None, @@ -192,6 +193,7 @@ def save( asarray=True, returnas='blended', ), + pfe=pfe, sep=sep, path=path, name=name, diff --git a/datastock/_saveload.py b/datastock/_saveload.py index 223cdd5..1524c42 100644 --- a/datastock/_saveload.py +++ b/datastock/_saveload.py @@ -26,6 +26,7 @@ def save( dflat=None, + pfe=None, sep=None, name=None, path=None, @@ -40,30 +41,89 @@ def save( # check inputs # ------------ - # path - path = _generic_check._check_var( - path, 'path', - default=os.path.abspath('./'), - types=str, - ) - path = os.path.abspath(path) - if not os.path.isdir(path): - msg = f"Arg path must be a valid path!\nProvided: {path}" + # ------------------ + # pfe vs path/name + + lc = [ + pfe is not None, + path is not None or name is not None, + ] + + if np.sum(lc) > 1: + msg = ( + "Saving, please provide {pfe} xor {path and/or name}!\n" + f"\t- path: {path}\n" + f"\t- name: {name}\n" + f"\t- pfe: {pfe}\n" + ) raise Exception(msg) - # clsname - clsname = _generic_check._check_var( - clsname, 'clsname', - default='DataCollection', - types=str, - ) + # ------------------ + # pfe vs path/name - # name - name = _generic_check._check_var( - name, 'name', - default='name', - types=str, - ) + if lc[0]: + if not isinstance(pfe, str): + msg = ( + "Arg pfe must be a str ponting to a file!\n" + f"Provided: {pfe}\n" + ) + raise Exception(msg) + + sdir, sfile = os.path.split(pfe) + if sdir == '': + sdir = os.path.asbpath('.') + + # check path + if not os.path.isdir(sdir): + msg = ( + "Arg pfe seems to have a non-valid path!\n" + "Provided: {sdir}\n" + ) + raise Exception(msg) + + # check file name + if not sfile.endswith('.npz'): + sfile = f"{sfile}.npz" + + # re-assemble + pfe = os.path.join(sdir, sfile) + + else: + # path + path = _generic_check._check_var( + path, 'path', + default=os.path.abspath('./'), + types=str, + ) + path = os.path.abspath(path) + if not os.path.isdir(path): + msg = f"Arg path must be a valid path!\nProvided: {path}" + raise Exception(msg) + + # clsname + clsname = _generic_check._check_var( + clsname, 'clsname', + default='DataCollection', + types=str, + ) + + # name + name = _generic_check._check_var( + name, 'name', + default='name', + types=str, + ) + + # set automatic name + user = getpass.getuser() + dt = dtm.datetime.now().strftime("%Y%m%d-%H%M%S") + name = f'{clsname}_{name}_{user}_{dt}.npz' + + # pfe + pfe = os.path.join(path, name) + + # ------------------ + # options # overwrite overwrite = _generic_check._check_var( @@ -90,17 +150,12 @@ def save( # save / print / return # ---------------------- - user = getpass.getuser() - dt = dtm.datetime.now().strftime("%Y%m%d-%H%M%S") - name = f'{clsname}_{name}_{user}_{dt}.npz' - # add sep dflat[_KEY_SEP] = sep - # pfe - pfe = os.path.join(path, name) - + # ----------------- # check vs existing + if os.path.isfile(pfe): if overwrite is True: msg = ( @@ -116,15 +171,21 @@ def save( ) raise Exception(msg) + # -------- # save - np.savez(pfe, **dflat) + np.savez(pfe, **dflat) + + # ------- # print + if verb: msg = f"\nSaved in:\n\t{pfe}\n" print(msg) + # ------- # return + if return_pfe is True: return pfe From 408baa6add0ec7775af96a8d718fbcd5d1858b42 Mon Sep 17 00:00:00 2001 From: Didier Vezinet Date: Wed, 29 Jan 2025 16:29:35 -0500 Subject: [PATCH 4/5] [#181] Added unit tests for save(pfe=str) --- datastock/tests/test_01_DataStock.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/datastock/tests/test_01_DataStock.py b/datastock/tests/test_01_DataStock.py index 0ebd01e..e1d479a 100644 --- a/datastock/tests/test_01_DataStock.py +++ b/datastock/tests/test_01_DataStock.py @@ -602,7 +602,12 @@ def test21_copy_equal(self): def test22_get_nbytes(self): nb, dnb = self.st.get_nbytes() - def test23_saveload(self, verb=False): + def test23_save_pfe(self, verb=False): + pfe = os.path.join(_PATH_OUTPUT, 'testsave.npz') + pfe = self.st.save(pfe=pfe, return_pfe=False) + os.remove(pfe) + + def test24_saveload(self, verb=False): pfe = self.st.save(path=_PATH_OUTPUT, verb=verb, return_pfe=True) st2 = load(pfe, verb=verb) # Just to check the loaded version works fine From 957c6d0a72a89b0704e05f2a8460b0fdbef0d05a Mon Sep 17 00:00:00 2001 From: Didier Vezinet Date: Wed, 29 Jan 2025 16:34:37 -0500 Subject: [PATCH 5/5] [#181] Debug unit tests for save(pfe=str) --- datastock/tests/test_01_DataStock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datastock/tests/test_01_DataStock.py b/datastock/tests/test_01_DataStock.py index e1d479a..2b2213d 100644 --- a/datastock/tests/test_01_DataStock.py +++ b/datastock/tests/test_01_DataStock.py @@ -604,7 +604,7 @@ def test22_get_nbytes(self): def test23_save_pfe(self, verb=False): pfe = os.path.join(_PATH_OUTPUT, 'testsave.npz') - pfe = self.st.save(pfe=pfe, return_pfe=False) + self.st.save(pfe=pfe, return_pfe=False) os.remove(pfe) def test24_saveload(self, verb=False):