time formatting

Author: dmnfarrell

CHANGES

@@ -6,7 +6,9 @@
 * progress bar for saving
 * filtering tool
 * snap works
-* fixes to import dialog 
+* fixes to import dialog
+* can now set datetime display format
+* show memory usage in status bar
 
 -----
 0.2.0

setup.py

@@ -16,7 +16,7 @@
     author_email = 'farrell.damien@gmail.com',
     packages = ['tablexplore'],
     package_data={'tablexplore': ['logo.png', '../description.txt',
-                                  'styles/*.css','icons/*.png',
+                                  'styles/*.qss','icons/*.png',
                                   'datasets/*.csv']},
     install_requires=['matplotlib>=3.0',
                       'pandas>=1.1',

snapcraft.yaml

@@ -1,16 +1,16 @@
-name: tablexplore
+name: Tablexplore
 version: '0.3.0'
 #version-script: git describe --abbrev=1 --tags
 summary: data plotting and analysis package
 description: |
-  tablexplore is an open source desktop application for data analysis and plotting
+  Tablexplore is an open source desktop application for data analysis and plotting
   intended for use in both research and education. The program allows quick
   visualization of data, table manipulation tools and supports large data tables.
 
 base: core18
 grade: stable
 confinement: strict
-icon: snap/gui/icon.png
+icon: img/logo.svg
 
 apps:
   tablexplore:

tablexplore/app.py

@@ -117,6 +117,7 @@ def loadSettings(self):
             core.FONT = s.value("font")
             core.FONTSIZE = int(s.value("fontsize"))
             core.COLUMNWIDTH = int(s.value("columnwidth"))
+            core.TIMEFORMAT = s.value("timeformat")
             r = s.value("recent_files")
             if r != '':
                 self.recent_files = r.split(',')
@@ -125,7 +126,6 @@ def loadSettings(self):
                 self.recent_urls = r.split('^^')
         except:
             pass
-
         return
 
     def saveSettings(self):
@@ -137,6 +137,7 @@ def saveSettings(self):
         self.settings.setValue('columnwidth', core.COLUMNWIDTH)
         self.settings.setValue('font', core.FONT)
         self.settings.setValue('fontsize', core.FONTSIZE)
+        self.settings.setValue('timeformat', core.TIMEFORMAT)
         self.settings.setValue('recent_files',','.join(self.recent_files))
         self.settings.setValue('recent_urls','^^'.join(self.recent_urls))
         if hasattr(self, 'plotgallery'):
@@ -166,7 +167,7 @@ def createToolBar(self):
                  'zoom in': {'action':self.zoomIn,'file':'zoom-in'},
                  'decrease columns': {'action': lambda: self.changeColumnWidths(.9),'file':'decrease-width'},
                  'increase columns': {'action': lambda: self.changeColumnWidths(1.1),'file':'increase-width'},
-                 'add sheet': {'action':self.addSheet,'file':'add'},
+                 'add sheet': {'action': lambda: self.addSheet(name=None),'file':'add'},
                  #'lock': {'action':self.lockTable,'file':'lock'},
                  'clean data': {'action':lambda: self._call('cleanData'),'file':'clean'},
                  'table to text': {'action':lambda: self._call('showAsText'),'file':'tabletotext'},
@@ -372,7 +373,7 @@ def openProject(self, filename=None, asksave=False):
         if filename == None:
             options = QFileDialog.Options()
             filename, _ = QFileDialog.getOpenFileName(self,"Open Project",
-                                  "","tablexplore Files (*.txpl);;All files (*.*)",
+                                  homepath,"tablexplore Files (*.txpl);;All files (*.*)",
                                   options=options)
 
         if not filename:
@@ -405,14 +406,15 @@ def saveAsProject(self):
 
         options = QFileDialog.Options()
         filename, _ = QFileDialog.getSaveFileName(self,"Save Project",
-                                                  "","tablexplore Files (*.txpl);;All files (*.*)",
+                                                  homepath,"tablexplore Files (*.txpl);;All files (*.*)",
                                                   options=options)
         if not filename:
             return
 
         self.filename = filename
         self.do_saveProject(filename)
         self.addRecentFile(filename)
+        self.proj_label.setText(self.filename)
         return
 
     def saveProject(self, filename=None):
@@ -614,16 +616,20 @@ def exportAs(self):
     def addSheet(self, name=None, df=None, meta=None):
         """Add a new sheet"""
 
-        names = self.sheets.keys()
-        if name is None or name in self.sheets:
-            name = 'dataset'+str(len(self.sheets)+1)
+        names = list(self.sheets.keys())
+        i=len(self.sheets)+1
+        if name == None or name in names:
+            name = 'dataset'+str(i)
+        if name in names:
+            import random
+            name = 'dataset'+str(random.randint(i,100))
 
         sheet = QSplitter(self.main)
         idx = self.main.addTab(sheet, name)
         #provide reference to self to dataframewidget
         dfw = DataFrameWidget(sheet, dataframe=df, app=self,
                                 font=core.FONT, fontsize=core.FONTSIZE,
-                                columnwidth=core.COLUMNWIDTH)
+                                columnwidth=core.COLUMNWIDTH, timeformat=core.TIMEFORMAT)
         sheet.addWidget(dfw)
 
         self.sheets[name] = dfw
@@ -658,6 +664,10 @@ def renameSheet(self):
         new, ok = QInputDialog.getText(self, 'New name', 'Name:',
                     QLineEdit.Normal, name)
         if ok:
+            if new in self.sheets:
+                QMessageBox.information(self, "Cannot rename",
+                                    "Sheet name already present")
+                return
             self.sheets[new] = self.sheets[name]
             del self.sheets[name]
             self.main.setTabText(index, new)
@@ -843,7 +853,7 @@ def preferences(self):
 
         from . import dialogs
         opts = {'font':core.FONT, 'fontsize':core.FONTSIZE,
-                'columnwidth':core.COLUMNWIDTH}
+                'columnwidth':core.COLUMNWIDTH, 'timeformat':core.TIMEFORMAT}
         dlg = dialogs.PreferencesDialog(self, opts)
         dlg.exec_()
         return

tablexplore/core.py

@@ -24,6 +24,7 @@
 import tempfile
 import numpy as np
 import pandas as pd
+from pandas.api.types import is_datetime64_any_dtype as is_datetime
 import string
 from PySide2 import QtCore, QtGui
 from PySide2.QtCore import QObject, Signal, Slot
@@ -38,6 +39,7 @@
 FONTSIZE = 12
 FONTSTYLE = ''
 COLUMNWIDTH = 80
+TIMEFORMAT = '%m/%d/%Y'
 
 try:
     _fromUtf8 = QtCore.QString.fromUtf8
@@ -59,6 +61,13 @@ def _fromUtf8(s):
          'subtable':'subtable','clear':'clear'
          }
 
+timeformats = ['infer','%d/%m/%Y','%d/%m/%y',
+                '%Y/%m/%d','%y/%m/%d','%Y/%d/%m',
+                '%d%m%Y','%Y%m%d','%Y%d%m',
+                '%d-%b-%Y',
+                '%Y-%m-%d %H:%M:%S','%Y-%m-%d %H:%M',
+                '%d-%m-%Y %H:%M:%S','%d-%m-%Y %H:%M']
+
 class ColumnHeader(QHeaderView):
     def __init__(self):
         super(QHeaderView, self).__init__()
@@ -323,21 +332,32 @@ def findDuplicates(self):
                 'tooltip':'Remove duplicates'},
                 'useselected':  {'type':'checkbox','default':0,'label':'Use selected columns' },
                 'keep':{'label':'Keep','type':'combobox','default':'first',
-                             'items':['first','last'], 'tooltip':'values to keep'}
+                             'items':['first','last'], 'tooltip':'values to keep'},
+                'inplace':{'type':'checkbox','default':0,'label':'In place' },
                 }
         dlg = dialogs.MultipleInputDialog(self, opts, title='Clean Data')
         dlg.exec_()
         if not dlg.accepted:
             return
         kwds = dlg.values
-        cols = df.columns
         keep = kwds['keep']
         remove = kwds['remove']
+        inplace = kwds['inplace']
+        if kwds['useselected'] == 1:
+            idx = self.table.getSelectedColumns()
+            cols = df.columns[idx]
+        else:
+            cols = df.columns
+
         new = df[df.duplicated(subset=cols,keep=keep)]
         if remove == True:
-            self.table.model.df = df.drop_duplicates(subset=cols,keep=keep)
-            self.refresh()
-        if len(new)>0:
+            new = df.drop_duplicates(subset=cols,keep=keep)
+            if inplace == True:
+                self.table.model.df = new
+                self.refresh()
+            elif len(new)>0:
+                self.showSubTable(new)
+        else:
             self.showSubTable(new)
         return
 
@@ -723,13 +743,11 @@ def convertDates(self, column):
             colname = '-'.join(cols)
             temp = df[cols]'''
 
-        timeformats = ['infer','%d/%m/%Y','%Y/%m/%d','%Y/%d/%m',
-                        '%Y-%m-%d %H:%M:%S','%Y-%m-%d %H:%M',
-                        '%d-%m-%Y %H:%M:%S','%d-%m-%Y %H:%M']
         props = ['','day','dayofweek','month','hour','minute','second','microsecond','year',
                  'dayofyear','weekofyear','quarter','days_in_month','is_leap_year']
-        opts = {'format':  {'type':'combobox','default':'int',
+        opts = {'format':  {'type':'combobox','default':'int','editable':True,
                             'items':timeformats,'label':'Conversion format'},
+                'errors':{'type':'combobox','items':['ignore','coerce'],'default':'ignore','label':'Errors'},
                 'prop':  {'type':'combobox','default':'int',
                         'items':props,'label':'Extract from datetime'} }
 
@@ -741,12 +759,16 @@ def convertDates(self, column):
 
         format = kwds['format']
         prop = kwds['prop']
+        errors = kwds['errors']
+        infer=False
         if format == 'infer':
             format = None
-
+            infer = True
         temp = df[column]
+        self.table.storeCurrent()
         if temp.dtype != 'datetime64[ns]':
-            temp = pd.to_datetime(temp, format=format)
+            temp = pd.to_datetime(temp, format=format, infer_datetime_format=infer,
+                                errors=errors)
 
         if prop != '':
             new = getattr(temp.dt, prop)
@@ -1015,13 +1037,14 @@ class DataFrameTable(QTableView):
     QTableView with pandas DataFrame as model.
     """
     def __init__(self, parent=None, dataframe=None, font='Arial',
-                    fontsize=12, columnwidth=80, align=None, **kwargs):
+                    fontsize=12, columnwidth=80, timeformat='%m-%d-%Y', **kwargs):
 
         QTableView.__init__(self)
         self.parent = parent
         self.font = font
         self.fontsize = fontsize
-        self.columnwidth=columnwidth
+        self.columnwidth = columnwidth
+        self.timeformat = timeformat
         self.clicked.connect(self.showSelection)
         #self.doubleClicked.connect(self.handleDoubleClick)
         #self.setSelectionBehavior(QTableView.SelectRows)
@@ -1126,7 +1149,7 @@ def getMemory(self):
 
         m = self.model.df.memory_usage(deep=True).sum()
         if m>1e5:
-            m = int(m/1048576)
+            m = round(m/1048576,2)
             units='MB'
         else:
             units='Bytes'
@@ -1507,9 +1530,12 @@ def data(self, index, role=QtCore.Qt.DisplayRole):
         i = index.row()
         j = index.column()
         coltype = self.df.dtypes[j]
+        isdate = is_datetime(coltype)
         if role == QtCore.Qt.DisplayRole:
             value = self.df.iloc[i, j]
-            if type(value) != str:
+            if isdate:
+                return value.strftime(TIMEFORMAT)
+            elif type(value) != str:
                 if type(value) in [float,np.float64] and np.isnan(value):
                     return ''
                 elif type(value) == np.float:

tablexplore/dialogs.py

@@ -914,16 +914,24 @@ def createWidgets(self, options):
 
         import pylab as plt
         colormaps = sorted(m for m in plt.cm.datad if not m.endswith("_r"))
+        timeformats = ['%m/%d/%Y','%d/%m/%Y','%d/%m/%y',
+                '%Y/%m/%d','%y/%m/%d','%Y/%d/%m',
+                '%d-%b-%Y','%b-%d-%Y',
+                '%Y-%m-%d %H:%M:%S','%Y-%m-%d %H:%M',
+                '%d-%m-%Y %H:%M:%S','%d-%m-%Y %H:%M']
         self.opts = {'rowheight':{'type':'spinbox','default':18,'range':(5,50),'label':'row height'},
                 'columnwidth':{'type':'spinbox','range':(10,300),
                 'default': options['columnwidth'], 'label':'column width'},
                 'alignment':{'type':'combobox','default':'w','items':['left','right','center'],'label':'text align'},
                 'font':{'type':'font','default':'Arial','default':options['font']},
-                'fontsize':{'type':'slider','default':options['fontsize'],'range':(5,40),'interval':1,'label':'font size'},
-                'floatprecision':{'type':'spinbox','default':2, 'label':'precision'},
+                'fontsize':{'type':'slider','default':options['fontsize'],'range':(5,40),
+                            'interval':1,'label':'font size'},
+                'timeformat':{'type':'combobox','default':options['timeformat'],
+                            'items':timeformats,'label':'Date/Time format'}
+                #'floatprecision':{'type':'spinbox','default':2, 'label':'precision'},
                 }
         sections = {'table':['alignment','rowheight','columnwidth'],
-                    'formats':['font','fontsize','floatprecision']}
+                    'formats':['font','fontsize','timeformat']}
 
         dialog, self.widgets = dialogFromOptions(self, self.opts, sections)
 
@@ -954,15 +962,10 @@ def apply(self):
         core.FONT = kwds['font']
         core.FONTSIZE = kwds['fontsize']
         core.COLUMNWIDTH = kwds['columnwidth']
+        core.TIMEFORMAT = kwds['timeformat']
         self.parent.refresh()
         return
 
-    def save(self):
-        """Save from current dialog settings"""
-
-        cp.write(open(default_conf,'w'))
-        return
-
 class FilterDialog(QWidget):
     """Qdialog for table query/filtering"""
     def __init__(self, parent, table, title=None):