merging gmv-1.9.1 in master

Author: gaubert

.gitignore

@@ -2,6 +2,7 @@
 *.pyc
 *.o
 *.class
+*.spec
 gmvault.log
 .DS_Store
 dists/disstoolbox-dist/build

Makefile

@@ -26,8 +26,11 @@ PYTHONBIN=python #MacOSX machine
 PYTHONWINBIN=/c/Python27/python.exe #windows laptop
 PYTHONVERSION=2.7
 
+PYINSTALLERMAC=/Library/Frameworks/Python.framework/Versions/2.7/bin/pyinstaller
+PYINSTALLERWIN=c:/Python27/Scripts/pyinstaller.exe
+
 #MAKENSIS=/cygdrive/d/Programs/NSIS/makensis.exe #windows work
-MAKENSIS=/c/Program\ Files/NSIS/makensis.exe #windows laptop
+MAKENSIS=c:/Program\ Files/NSIS/makensis.exe #windows laptop
 
 #VERSION is in gmv_cmd.py as GMVAULT_VERSION
 GMVVERSION=$(shell python $(BASEDIR)/etc/utils/find_version.py $(BASEDIR)/src/gmv/gmvault_utils.py)
@@ -123,15 +126,9 @@ gmv-linux-dist: clean init
 	@echo "=================================================================="
 
 gmv-mac-dist: clean init
-	cp -R $(BASEDIR)/src/gmv $(GMVDIST)
-	cp -R $(BASEDIR)/src/gmv_runner.py $(GMVDIST)
-	cp $(BASEDIR)/src/setup_mac.py $(GMVDIST)
-	cd $(GMVDIST); $(PYTHONBIN) setup_mac.py py2app --arch=intel
-	mkdir -p $(GMVDIST)/$(GMVDISTNAME)/bin ; mkdir -p $(GMVDIST)/$(GMVDISTNAME)/lib
-	cp $(BASEDIR)/etc/scripts/gmvault_mac $(GMVDIST)/$(GMVDISTNAME)/bin/gmvault
-	cp -R $(BASEDIR)/README.md $(GMVDIST)/$(GMVDISTNAME)/bin/README.txt
-	cp $(BASEDIR)/RELEASE-NOTE.txt $(GMVDIST)/$(GMVDISTNAME)/bin/RELEASE-NOTE.txt
-	cp -R $(GMVDIST)/dist/gmv_runner.app $(GMVDIST)/$(GMVDISTNAME)/lib
+	$(PYINSTALLERMAC) --onefile --name gmvault --distpath=$(GMVDIST)/$(GMVDISTNAME)  $(BASEDIR)/src/gmv_runner.py
+	cp -R $(BASEDIR)/README.md $(GMVDIST)/$(GMVDISTNAME)/README.txt
+	cp $(BASEDIR)/RELEASE-NOTE.txt $(GMVDIST)/$(GMVDISTNAME)/RELEASE-NOTE.txt
 	cd $(GMVDIST); tar zcvf ./$(GMVDISTNAME)-macosx-intel.tar.gz ./$(GMVDISTNAME)
 	@echo ""
 	@echo "========================================="
@@ -140,12 +137,8 @@ gmv-mac-dist: clean init
 	@echo ""
 	@echo "========================================="
 
-gmv-win-dist: init 
-	mkdir -p $(GMVWINBUILDDIST)
-	cp -R $(BASEDIR)/src/gmv $(GMVDIST)
-	cp -R $(BASEDIR)/src/gmv_runner.py $(GMVDIST)
-	cp $(BASEDIR)/src/setup_win.py $(GMVDIST)
-	cd $(GMVDIST); $(PYTHONWINBIN) setup_win.py py2exe -d ../$(GMVWINBUILDDIST)
+gmv-win-dist: clean init
+	$(PYINSTALLERWIN) --onefile --name gmv_runner --distpath=$(GMVWINBUILDDIST) $(BASEDIR)/src/gmv_runner.py
 	cp $(BASEDIR)/etc/scripts/gmvault.bat $(GMVWINBUILDDIST)
 	cp $(BASEDIR)/etc/scripts/gmvault-shell.bat $(GMVWINBUILDDIST)
 	cd .; $(PYTHONWINBIN) $(BASEDIR)/etc/utils/add_version.py $(BASEDIR)/etc/scripts/gmv-msg.bat $(GMVWINBUILDDIST)/gmv-msg.bat $(GMVVERSION)
@@ -168,12 +161,11 @@ gmv-win-installer: gmv-win-dist
 
 clean: clean-build
 	mkdir -p $(GMVDIST)
-	cd $(GMVDIST); rm -Rf ./etc; rm -f MANIFEST.in; rm -f RELEASE-NOTE.txt; rm -Rf build; rm -Rf gmvault.egg-info; rm -f setup*.py ; rm -Rf dist ; rm -Rf src; rm -f README* ;rm -Rf GMVault.egg-info; rm -Rf gmv; rm -Rf scripts; rm -f *.tar.gz
+	rm -Rf $(GMVDIST)/*
 
 clean-build:
 	mkdir -p $(GMVBUILD)
-	cd $(GMVBUILD); rm -Rf egg-dist; 
+	rm -Rf $(GMVBUILD)/egg-dist; 
 	rm -Rf $(GMVDIST)/$(GMVDISTNAME)
 	rm -Rf $(GMVWINBUILDDIST)
 
-    

RELEASE-NOTE.txt

@@ -1,6 +1,37 @@
 Release Notes for the different versions:
 -------------------------------
 
+-------------------------------
+Version Gmvault-v1.9.1-beta:
+-------------------------------
+-------------------------------
+Date: 31.03.2016
+
+Release fixing many bugs left in 1.0-beta and improving deployment on Mac OS X and Windows (use now pyinstaller).
+
+Bug Fixes:
+----------
+- Fix bug preventing to use encryption [Github issue #84]
+- Fix many character encoding bugs in emails [Github issue #211, #213, #152, #218, ] 
+- Fix encoding bug in labels [Github issue #180] 
+- Fix encoding bug in commandline parameters [Github issue #195]
+- Fix bug preventing email containing null characters to be restored [Github #212]
+- Fix bug corrupting oauth2 credential when saved in file [Github #209]
+
+Shoutouts to: 
+-------------
+
+Shout out to everybody who continued supporting the project while I was away.
+
+Also shoutouts to all the users that sent me support emails and raised issues by emails.
+
+Gmvault Pantheon:
+-----------------
+
+A million time thank you for the benevolent donors that are forever in the Gmvault Pantheon and All of Fame for their gesture.
+
+Jason Stangroome, Mathieu Pinson, Mathieu Clavel, Stijn Jonker, Douglas Carter, Steven Moschidis, GoVelvet LLC, Daniel Brass , Joseph Colburn, Joanne Kaye, Bernie Santos, Wolfgang Schwaiger, Kraut Computing, Matt Burke, Ademario Araujo Jr, Roque Pinel, George Armhold, Randi Veiberg, Gregory Szilagyi, Antony Moore, Filip Moerman, Axel Amigo Arnold, Jonathan Sarasin, Kirby Moyers, Alberto Cottica, Richard Lee, Suneel Kanuri, Owen Phelps, Clint Chapman, Donmez Ismail, Tomas Bäckman, Keith Petersen, Holden Karau, James Mcfarland, Clayton Stanley, Stefan Habetz, Francesco Bellomo, Brian Yanosik, Bjarte Aune Olsen, Joel Sylvester, Billy Amon, Alexander Halperin, John Zendano, Alex Van't Hof, JJ Tiziou Photography, Sally Carr, Jacek Szczech, Baden Holt, Eleonora Borelli, Patrick Van Humbeeck
+
 -------------------------------
 Version Gmvault-v1.9-beta:
 -------------------------------

etc/nsis-install/gmvault_setup.nsi

@@ -118,11 +118,11 @@ File gmv-icon.ico
 File gmvault-shell.bat
 File RELEASE-NOTE.txt
 File README.txt
-File library.zip
-File python27.dll
-File w9xpopen.exe
-File *.pyd
-File /r Microsoft.VC90.CRT
+;File library.zip
+;File python27.dll
+;File w9xpopen.exe
+;File *.pyd
+;File /r Microsoft.VC90.CRT
 
 IfErrors 0 +2
 
@@ -149,13 +149,13 @@ Delete $INSTDIR\library.zip
 Delete $INSTDIR\*.ico
 Delete $INSTDIR\*.bat
 Delete $INSTDIR\*.txt
-Delete $INSTDIR\python27.dll
-Delete $INSTDIR\w9xpopen.exe
-Delete $INSTDIR\*.pyd
-Delete $INSTDIR\Microsoft.VC90.CRT\*.dll
-Delete $INSTDIR\Microsoft.VC90.CRT\*.manifest
+;Delete $INSTDIR\python27.dll
+;Delete $INSTDIR\w9xpopen.exe
+;Delete $INSTDIR\*.pyd
+;Delete $INSTDIR\Microsoft.VC90.CRT\*.dll
+;Delete $INSTDIR\Microsoft.VC90.CRT\*.manifest
 Delete $INSTDIR\${UNINSTALLER_NAME}
-rmDir /r $INSTDIR\Microsoft.VC90.CRT
+;rmDir /r $INSTDIR\Microsoft.VC90.CRT
 ;rmDir /r $INSTDIR
 rmDir $INSTDIR
 

setup.py

@@ -67,5 +67,5 @@ def find_version(path):
       package_data={'': ['release-note.txt']},
       include_package_data=True,
       #install_requires=['argparse', 'Logbook==0.4.1', 'IMAPClient==0.9.2','gdata==2.0.17']
-      install_requires=['argparse', 'Logbook==0.9.0', 'IMAPClient==0.12', 'chardet==2.3.0']
+      install_requires=['argparse', 'Logbook==0.10.1', 'IMAPClient==0.13', 'chardet==2.3.0']
       )

src/gmv/credential_utils.py

@@ -23,7 +23,7 @@
 import webbrowser
 import json
 import base64
-import urllib
+import urllib #for urlencode
 import urllib2
 
 import os
@@ -69,7 +69,7 @@ def get_secret_key(cls, a_filepath):
            Get secret key if it is in the file otherwise generate it and save it
         """
         if os.path.exists(a_filepath):
-            with open(a_filepath).read() as f:
+            with open(a_filepath) as f:
                 secret = f.read()
         else:
             secret = gmvault_utils.make_password()
@@ -114,7 +114,14 @@ def store_oauth2_credentials(cls, email, access_token, refresh_token, validity,
 
         # Open a file
         fdesc = os.open(oauth_file, os.O_RDWR|os.O_CREAT )
-        fobj = os.fdopen(fdesc, "w+")
+
+        #write new content
+        fobj = os.fdopen(fdesc, "w")
+
+        #empty file
+        fobj.truncate()
+        fobj.seek(0, os.SEEK_SET)
+
 
         the_obj = { "access_token"    : access_token,
                     "refresh_token"   : refresh_token,
@@ -293,7 +300,7 @@ def _get_authorization_tokens(cls, authorization_code):
         request_url = '%s/%s' % (account_base_url, 'o/oauth2/token')
 
         try:
-            response = urllib.urlopen(request_url, urllib.urlencode(params)).read()
+            response = urllib2.urlopen(request_url, urllib.urlencode(params)).read()
         except Exception, err: #pylint: disable-msg=W0703
             LOG.critical("Error: Problems when trying to connect to Google oauth2 endpoint: %s." % (request_url))
             raise err

src/gmv/gmv_cmd.py

@@ -149,7 +149,7 @@ class GMVaultLauncher(object):
     def __init__(self):
         """ constructor """
         super(GMVaultLauncher, self).__init__()
-        
+
     @gmvault_utils.memoized
     def _create_parser(self): #pylint: disable=R0915
         """
@@ -201,6 +201,10 @@ def _create_parser(self): #pylint: disable=R0915
                           help="use interactive password authentication, encrypt and store the password. (not recommended)",
                           action= 'store_const' , dest="passwd", const='store')
 
+        #sync_parser.add_argument("-r", "--imap-req", type = get_unicode_commandline_arg, metavar = "REQ", \
+        #                         help="Imap request to restrict sync.",\
+        #                         dest="imap_request", default=None)
+
         sync_parser.add_argument("-r", "--imap-req", metavar = "REQ", \
                                  help="Imap request to restrict sync.",\
                                  dest="imap_request", default=None)
@@ -273,7 +277,7 @@ def _create_parser(self): #pylint: disable=R0915
                                  default='full', help='type of restoration: full|quick. (default: full)')
 
         # add a label
-        rest_parser.add_argument('-a', '--apply-label', \
+        rest_parser.add_argument('-a', '--apply-label' , \
                                  action='store', dest='apply_label', \
                                  default=None, help='Apply a label to restored emails')
 
@@ -694,7 +698,8 @@ def _sync(cls, args, credential):
         elif args.get('type', '') == 'custom':
 
             #convert args to unicode
-            args['request']['req']     = gmvault_utils.convert_to_unicode(args['request']['req'])
+            u_str = gmvault_utils.convert_argv_to_unicode(args['request']['req'])
+            args['request']['req']     = u_str
             args['request']['charset'] = 'utf-8' #for the moment always utf-8
             args['request']['mode']    = 'custom'
 
@@ -836,7 +841,7 @@ def bootstrap_run():
     """ temporary bootstrap """
 
     init_logging()
-    
+
     #force argv[0] to gmvault
     sys.argv[0] = "gmvault"
 
@@ -845,7 +850,7 @@ def bootstrap_run():
     gmvlt = GMVaultLauncher()
 
     args = gmvlt.parse_args()
-    
+
     #activate debug if enabled
     if args['debug']:
         LOG.critical("Activate debugging information.")

src/gmv/gmvault.py

@@ -102,7 +102,7 @@ def handle_sync_imap_error(the_exception, the_id, error_report, src):
         except Exception, _: #pylint:disable-msg=W0703
             curr = None
             LOG.critical("Error when trying to get gmail id for message with imap id %s." % (the_id))
-            LOG.critical("Disconnect, wait for 20 sec then reconnect.")
+            LOG.critical("Disconnect, wait for 10 sec then reconnect.")
             src.disconnect()
             #could not fetch the gm_id so disconnect and sleep
             #sleep 10 sec
@@ -265,7 +265,8 @@ def __init__(self, db_root_dir, host, port, login, \
         self.error_report = { 'empty' : [] ,
                               'cannot_be_fetched'  : [],
                               'emails_in_quarantine' : [],
-                              'reconnections' : 0}
+                              'reconnections' : 0,
+                              'key_error' : []}
 
         #instantiate gstorer
         self.gstorer =  gmvault_db.GmailStorer(self.db_root_dir, self.use_encryption)
@@ -290,14 +291,16 @@ def get_operation_report(self):
                   "%s operation performed in %s.\n" \
                   "Number of reconnections: %d.\nNumber of emails quarantined: %d.\n" \
                   "Number of emails that could not be fetched: %d.\n" \
-                  "Number of emails that were returned empty by gmail: %d\n"\
+                  "Number of emails that were returned empty by gmail: %d.\n"\
+                  "Number of emails without label information returned by gmail: %d.\n"\
                   "================================================================" \
               % (self.error_report['operation'], \
                  self.error_report['operation_time'], \
                  self.error_report['reconnections'], \
                  len(self.error_report['emails_in_quarantine']), \
                  len(self.error_report['cannot_be_fetched']), \
-                 len(self.error_report['empty'])
+                 len(self.error_report['empty']), \
+                 len(self.error_report['key_error'])
                 )
 
         LOG.debug("error_report complete structure = %s" % (self.error_report))
@@ -430,8 +433,9 @@ def _common_sync(self, a_timer, a_type, imap_req, compress, restart):
         """
         # get all imap ids in All Mail
         imap_ids = self.src.search(imap_req)
-       
-        last_id_file =  self.OP_EMAIL_SYNC if a_type == "email" else self.OP_CHAT_SYNC
+
+        last_id_file = self.OP_EMAIL_SYNC if a_type == "email" else self.OP_CHAT_SYNC
+        
         # check if there is a restart
         if restart:
             LOG.critical("Restart mode activated for emails. Need to find information in Gmail, be patient ...")
@@ -484,8 +488,23 @@ def _common_sync(self, a_timer, a_type, imap_req, compress, restart):
                     LOG.critical("Process %s num %d (imap_id:%s) from %s." % (a_type, nb_msgs_processed, the_id, the_dir))
 
                     #decode the labels that are received as utf7 => unicode
-                    new_data[the_id][imap_utils.GIMAPFetcher.GMAIL_LABELS] = \
-                    imap_utils.decode_labels(new_data[the_id][imap_utils.GIMAPFetcher.GMAIL_LABELS])
+                    try:
+                        new_data[the_id][imap_utils.GIMAPFetcher.GMAIL_LABELS] = \
+                             imap_utils.decode_labels(new_data[the_id][imap_utils.GIMAPFetcher.GMAIL_LABELS])
+                    except KeyError, ke:
+                        LOG.info("KeyError, reason: %s. new_data[%s]=%s" % (str(ke), the_id, new_data.get(the_id)))
+                        # try to fetch it individually and replace current info if it fails then raise error.
+                        id_info = None
+                        try:
+                            id_info = batch_fetcher.individual_fetch(the_id)
+                            new_data[the_id][imap_utils.GIMAPFetcher.GMAIL_LABELS] = \
+                                imap_utils.decode_labels(id_info[imap_utils.GIMAPFetcher.GMAIL_LABELS])
+                        except Exception, err:
+                            LOG.debug("Error when trying to fetch again information for email id %s. id_info = %s. exception:(%s)" \
+                                      % (the_id, id_info, str(err)))
+                            LOG.info("Missing labels information for email id %s. Ignore it\n" % (the_id))
+                            self.error_report['key_error'].append((the_id, new_data.get(the_id)))
+                            continue
 
                     LOG.debug("metadata info collected: %s\n" % (new_data[the_id]))
 
@@ -998,7 +1017,11 @@ def restore_chats(self, extra_labels = [], restart = False): #pylint:disable=W01
                     self.src.apply_labels_to(labels_to_apply[label], [label]) 
             except Exception, err:
                 LOG.error("Problem when applying labels %s to the following ids: %s" %(label, labels_to_apply[label]), err)
-                if isinstance(err, imaplib.IMAP4.abort) and str(err).find("=> Gmvault ssl socket error: EOF") >= 0:
+                if isinstance(err, imap_utils.LabelError) and err.ignore() == True:
+                    LOG.critical("Ignore labelling: %s" % (err))
+                    LOG.critical("Disconnecting and reconnecting to restart cleanly.")
+                    self.src.reconnect() #reconnect
+                elif isinstance(err, imaplib.IMAP4.abort) and str(err).find("=> Gmvault ssl socket error: EOF") >= 0:
                     # if this is a Gmvault SSL Socket error ignore labelling and continue the restore
                     LOG.critical("Ignore labelling")
                     LOG.critical("Disconnecting and reconnecting to restart cleanly.")
@@ -1082,6 +1105,9 @@ def restore_emails(self, pivot_dir = None, extra_labels = [], restart = False):
             # unbury the metadata for all these emails
             for gm_id in group_imap_ids:    
                 try:
+
+                    LOG.debug("Unbury email with gm_id %s." % (gm_id))
+
                     email_meta, email_data = self.gstorer.unbury_email(gm_id)
 
                     LOG.critical("Pushing email body with id %s." % (gm_id))
@@ -1125,7 +1151,6 @@ def restore_emails(self, pivot_dir = None, extra_labels = [], restart = False):
 
             # associate labels with emails
             LOG.critical("Applying labels to the current batch of emails.")
-
             try:
                 LOG.debug("Changing directory. Going into ALLMAIL")
                 the_timer = gmvault_utils.Timer()
@@ -1136,8 +1161,11 @@ def restore_emails(self, pivot_dir = None, extra_labels = [], restart = False):
                     self.src.apply_labels_to(labels_to_apply[label], [label]) 
             except Exception, err:
                 LOG.error("Problem when applying labels %s to the following ids: %s" %(label, labels_to_apply[label]), err)
-                LOG.error("Problem when applying labels.", err)
-                if isinstance(err, imaplib.IMAP4.abort) and str(err).find("=> Gmvault ssl socket error: EOF") >= 0:
+                if isinstance(err, imap_utils.LabelError) and err.ignore() == True:
+                    LOG.critical("Ignore labelling: %s" % (err))
+                    LOG.critical("Disconnecting and reconnecting to restart cleanly.")
+                    self.src.reconnect() #reconnect
+                elif isinstance(err, imaplib.IMAP4.abort) and str(err).find("=> Gmvault ssl socket error: EOF") >= 0:
                     # if this is a Gmvault SSL Socket error ignore labelling and continue the restore
                     LOG.critical("Ignore labelling")
                     LOG.critical("Disconnecting and reconnecting to restart cleanly.")