Skip to content

Commit f0d4765

Browse files
JohnDN90JohnDN90
committed
Fixed bug, hard coded path, in folderStructureBackup.py.
Fixed bug, os.type to os.name, in folderStructureBackup.py. Removed some unncessary comments leftover from initial development. Created folderStructureBackup_linux.py.
1 parent 6bdda10 commit f0d4765

File tree

2 files changed

+292
-15
lines changed

2 files changed

+292
-15
lines changed

‎folderStructureBackup.py‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -105,11 +105,6 @@
105105
if not rootPath.endswith("\\"):
106106
rootPath = rootPath + "\\"
107107

108-
# backupPath = "F:\\"
109-
# rootPath = "C:\\"
110-
# storeHash = True
111-
# hashType = "xxhash"
112-
113108
# Get the current date and time
114109
currentDT = datetime.datetime.now()
115110
strDT = currentDT.strftime("%Y%m%d_%H%M%S")
@@ -136,7 +131,6 @@
136131

137132
# Credit: quantumSoup and awiebe on Stackoverflow.com
138133
def md5(fname):
139-
# 70.429 seconds
140134
hash_md5 = hashlib.md5()
141135
with open(fname, "rb") as f:
142136
for chunk in iter(lambda: f.read(4096), b""):
@@ -145,7 +139,6 @@ def md5(fname):
145139

146140

147141
def crc32(fname):
148-
# 64.230 seconds
149142
with open(fname, "rb") as f:
150143
value = 0
151144
for chunk in iter(lambda: f.read(4096), b""):
@@ -154,7 +147,6 @@ def crc32(fname):
154147

155148

156149
def xxHash(fname):
157-
# 62.890 seconds
158150
# pip install xxhash if necessary
159151
hash_xxhash = xxhash.xxh64()
160152
with open(fname, "rb") as f:
@@ -171,7 +163,7 @@ def xxHash(fname):
171163
elif hashType.lower() == "md5":
172164
getHash = md5
173165

174-
obj_Disk = psutil.disk_usage('F:\\')
166+
obj_Disk = psutil.disk_usage(rootPath)
175167
totalSize = float(obj_Disk.used)
176168

177169
start = time.time()
@@ -214,14 +206,15 @@ def xxHash(fname):
214206

215207
for name in dirs:
216208
val = os.path.join(root, name)
217-
# print(val)
218209
f.write("%s\n" % (val))
219210

220211
elif (os.name == "nt") and not (storeHash):
221212
with open(outputPath, 'w') as f:
222213
# Walk through all the directories and files in the rootPath specified above and write them to the backup file
223214
f.write("#%s\n" % (rootPath))
224215
f.write("#OS: %s\n" % (os.name))
216+
f.write("#Date: %s\n" % (currentDT.strftime("%Y%m%d")))
217+
f.write("#Time: %s\n" % (currentDT.strftime("%H%M%S")))
225218
info = win32api.GetVolumeInformation('F:\\')
226219
f.write("#Volume Name: %s\n" % (info[0]))
227220
f.write("#Volume Serial No.: %s\n" % (info[1]))
@@ -251,12 +244,9 @@ def xxHash(fname):
251244
f.write("#Directories\n")
252245
for name in dirs:
253246
val = os.path.join(root, name)
254-
# print(val)
255247
f.write("%s\n" % (val))
256-
257-
258248
else:
259-
raise NotImplementedError("Code for operating system of type %s is not implemented."%(os.type))
249+
raise NotImplementedError("Code for operating system of type %s is not implemented."%(os.name))
260250

261251
end = time.time()
262252
print("\nTook %.3f seconds.\n"%(end-start))
@@ -269,5 +259,4 @@ def xxHash(fname):
269259
# Remove the original uncompressed backup file
270260
os.remove(outputPath)
271261

272-
273262
time.sleep(5)

‎folderStructureBackup_linux.py‎

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,288 @@
1+
"""
2+
FolderStructureBackup create a backup of a directory's folder structure.
3+
4+
Copyright (C) 2019 David John Neiferd
5+
6+
This program is free software: you can redistribute it and/or modify
7+
it under the terms of the GNU General Public License as published by
8+
the Free Software Foundation, either version 3 of the License, or
9+
(at your option) any later version.
10+
11+
This program is distributed in the hope that it will be useful,
12+
but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14+
GNU General Public License for more details.
15+
16+
You should have received a copy of the GNU General Public License
17+
along with this program. If not, see <https://www.gnu.org/licenses/>.
18+
"""
19+
20+
21+
__author__ = "David John Neiferd"
22+
__copyright__ = "Copyright (C) 2019 David John Neiferd"
23+
__version__ = "0.1"
24+
__license__ = "GNU GPL v3"
25+
__license_folderStructureBackup__ = ("----------------------------------------------------------\n"
26+
"FolderStructureBackup create a backup of a directory's folder structure.\n"
27+
"\n"
28+
"Copyright (C) 2019 David John Neiferd\n"
29+
"\n"
30+
"This program is free software: you can redistribute it and/or modify\n"
31+
"it under the terms of the GNU General Public License as published by\n"
32+
"the Free Software Foundation, either version 3 of the License, or\n"
33+
"(at your option) any later version.\n"
34+
"\n"
35+
"This program is distributed in the hope that it will be useful,\n"
36+
"but WITHOUT ANY WARRANTY; without even the implied warranty of\n"
37+
"MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n"
38+
"GNU General Public License for more details.\n"
39+
"\n"
40+
"You should have received a copy of the GNU General Public License\n"
41+
"along with this program. If not, see <https://www.gnu.org/licenses/>.\n"
42+
"----------------------------------------------------------\n")
43+
44+
__license_xxHash__ = ("----------------------------------------------------------\n"
45+
'\nxxHash Library\n'
46+
'Copyright (c) 2012-2014, Yann Collet\n'
47+
'All rights reserved.\n'
48+
'\n'
49+
'Redistribution and use in source and binary forms, with or without modification,\n'
50+
'are permitted provided that the following conditions are met:\n'
51+
'\n'
52+
'* Redistributions of source code must retain the above copyright notice, this\n'
53+
' list of conditions and the following disclaimer.\n'
54+
'\n'
55+
'* Redistributions in binary form must reproduce the above copyright notice, this\n'
56+
' list of conditions and the following disclaimer in the documentation and/or\n'
57+
' other materials provided with the distribution.\n'
58+
'\n'
59+
'THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND\n'
60+
'ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\n'
61+
'WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n'
62+
'DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR\n'
63+
'ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n'
64+
'(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n'
65+
'LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON\n'
66+
'ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n'
67+
'(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n'
68+
'SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n'
69+
"----------------------------------------------------------\n\n")
70+
71+
print(__license_xxHash__)
72+
print(__license_folderStructureBackup__)
73+
74+
import os
75+
import zipfile
76+
import datetime
77+
import time
78+
import sys
79+
import psutil
80+
from subprocess import check_output
81+
82+
83+
def getVolumeInfo(path):
84+
val = check_output(['df', '-hT', path]).split("\n")[1].split(" ")
85+
val = [v for v in val if v != '']
86+
filesystem = val[0]
87+
fstype = val[1]
88+
fssize = val[2]
89+
fsused = val[3]
90+
fsavail = val[4]
91+
fsmount = val[6]
92+
93+
val = check_output(['udevadm', 'info', '--query=all', '--name=%s'%filesystem]).strip().split("\n")
94+
hddserial = [v.split('=')[-1] for v in val if "ID_SERIAL=" in v][0]
95+
fstype2 = [v.split('=')[-1] for v in val if "ID_FS_TYPE=" in v][0]
96+
fsuuid = [v.split('=')[-1] for v in val if "ID_FS_UUID=" in v][0]
97+
try:
98+
fslabel = [v.split('=')[-1] for v in val if "ID_FS_LABEL=" in v][0]
99+
except:
100+
fslabel = "N/A"
101+
return fslabel, fstype, fssize, fsused, fsavail, fsmount, hddserial, fstype2, fsuuid, val
102+
103+
104+
# Specify the path of the backup and the rootPath of the folder structure that will be backed up
105+
if len(sys.argv)==1:
106+
rootPath = raw_input("Specify the path to be backed up: ")
107+
backupPath = raw_input("Specify location to create the backup: ")
108+
hashType = raw_input("Specifiy the hash type (leave blank for None): ") or None
109+
if hashType is None:
110+
storeHash = False
111+
else:
112+
storeHash = True
113+
else:
114+
rootPath = sys.argv[1]
115+
backupPath = sys.argv[2]
116+
if len(sys.argv)>3:
117+
hashType = sys.argv[3]
118+
storeHash = True
119+
else:
120+
hashType = None
121+
storeHash = False
122+
123+
if not backupPath.endswith("/"):
124+
backupPath = backupPath + "/"
125+
126+
if not rootPath.endswith("/"):
127+
rootPath = rootPath + "/"
128+
129+
# Get the current date and time
130+
currentDT = datetime.datetime.now()
131+
strDT = currentDT.strftime("%Y%m%d_%H%M%S")
132+
133+
# Create a backup file with current date and time in it
134+
outputPath = backupPath + "folderStructureBackup_" + strDT + ".txt"
135+
136+
print("\nSummary\n----------\nBacking Up: %s\nTo: %s\nHash Type: %s\n------------------------------------"%(rootPath, outputPath[:-4]+".zip", hashType))
137+
ans = raw_input("Ensure above information is correct. Start backup? (yes/no): \n\n")
138+
139+
if ans.lower() != "yes":
140+
raise RuntimeError("User canceled operation.")
141+
142+
if hashType is None:
143+
pass
144+
elif hashType.lower() == "xxhash":
145+
import xxhash
146+
elif hashType.lower() == "crc32":
147+
import zlib
148+
elif hashType.lower() == "md5":
149+
import hashlib
150+
else:
151+
raise ValueError("Uknonwn hash type, %s, specified by user."%hashType)
152+
153+
# Credit: quantumSoup and awiebe on Stackoverflow.com
154+
def md5(fname):
155+
hash_md5 = hashlib.md5()
156+
with open(fname, "rb") as f:
157+
for chunk in iter(lambda: f.read(4096), b""):
158+
hash_md5.update(chunk)
159+
return hash_md5.hexdigest()
160+
161+
162+
def crc32(fname):
163+
with open(fname, "rb") as f:
164+
value = 0
165+
for chunk in iter(lambda: f.read(4096), b""):
166+
value = zlib.crc32(chunk, value) & 0xfffffff
167+
return value
168+
169+
170+
def xxHash(fname):
171+
# pip install xxhash if necessary
172+
hash_xxhash = xxhash.xxh64()
173+
with open(fname, "rb") as f:
174+
for chunk in iter(lambda: f.read(4096), b""):
175+
hash_xxhash.update(chunk)
176+
return hash_xxhash.hexdigest()
177+
178+
if hashType is None:
179+
pass
180+
elif hashType.lower() == "xxhash":
181+
getHash = xxHash
182+
elif hashType.lower() == "crc32":
183+
getHash = crc32
184+
elif hashType.lower() == "md5":
185+
getHash = md5
186+
187+
# obj_Disk = psutil.disk_usage(rootPath)
188+
# totalSize = float(obj_Disk.used)
189+
totalSize = float(check_output(['du', '-s', rootPath]).split("\t")[0])*1000.0
190+
191+
start = time.time()
192+
sumSize = 0
193+
# Windows Code
194+
if (os.name == "posix") and (storeHash):
195+
with open(outputPath, 'w') as f:
196+
# Walk through all the directories and files in the rootPath specified above and write them to the backup file
197+
f.write("#%s\n" % (rootPath))
198+
f.write("#OS: %s\n" % (os.name))
199+
f.write("#Date: %s\n"%(currentDT.strftime("%Y%m%d")))
200+
f.write("#Time: %s\n"%(currentDT.strftime("%H%M%S")))
201+
fslabel, fstype, fssize, fsused, fsavail, fsmount, hddserial, fstype2, fsuuid, val = getVolumeInfo(rootPath)
202+
f.write("#Volume Name: %s\n" % (fslabel))
203+
f.write("#Volume Serial No.: %s\n" % (hddserial))
204+
f.write("#Volume UUID: %s\n"%(fsuuid))
205+
f.write("#File System Type: %s, %s\n" % (fstype, fstype2))
206+
f.write("#Extra HDD Info Below\n")
207+
for v in val: f.write("#%s\n"%v)
208+
f.write("#File List Format: file_path, file_size, file_mtime, %s_hash\n" % (hashType))
209+
for root, dirs, files in os.walk(rootPath, topdown=True):
210+
f.write("#Files\n")
211+
for name in files:
212+
val = os.path.join(root, name)
213+
try:
214+
size = os.path.getsize(val)
215+
except:
216+
size = "N/A"
217+
try:
218+
mtime = time.strftime('%Y%m%d_%H%M%S', time.gmtime(os.path.getmtime(val)))
219+
except:
220+
size = 0
221+
try:
222+
md5hash = getHash(val)
223+
except:
224+
md5hash = "N/A"
225+
f.write("%s, %s, %s, %s\n" % (val, size, mtime, md5hash))
226+
sumSize += size
227+
duration = time.time() - start
228+
avgSpeed = (sumSize/duration)/1000000.0
229+
progress = sumSize / float(totalSize) * 100.0
230+
print("Progress: %.2f%% Average Speed: %.3f MB/s"%(progress, avgSpeed))
231+
232+
for name in dirs:
233+
val = os.path.join(root, name)
234+
f.write("%s\n" % (val))
235+
236+
elif (os.name == "posix") and not (storeHash):
237+
with open(outputPath, 'w') as f:
238+
# Walk through all the directories and files in the rootPath specified above and write them to the backup file
239+
f.write("#%s\n" % (rootPath))
240+
f.write("#OS: %s\n" % (os.name))
241+
f.write("#Date: %s\n" % (currentDT.strftime("%Y%m%d")))
242+
f.write("#Time: %s\n" % (currentDT.strftime("%H%M%S")))
243+
fslabel, fstype, fssize, fsused, fsavail, fsmount, hddserial, fstype2, fsuuid, val = getVolumeInfo(rootPath)
244+
f.write("#Volume Name: %s\n" % (fslabel))
245+
f.write("#Volume Serial No.: %s\n" % (hddserial))
246+
f.write("#Volume UUID: %s\n" % (fsuuid))
247+
f.write("#File System Type: %s, %s\n" % (fstype, fstype2))
248+
f.write("#Extra HDD Info Below\n")
249+
for v in val: f.write("#%s\n" % v)
250+
f.write("#File List Format: file_path, file_size, file_mtime\n")
251+
for root, dirs, files in os.walk(rootPath, topdown=True):
252+
f.write("#Files\n")
253+
for name in files:
254+
val = os.path.join(root, name)
255+
try:
256+
size = os.path.getsize(val)
257+
except:
258+
size = 0
259+
try:
260+
mtime = time.strftime('%Y%m%d_%H%M%S', time.gmtime(os.path.getmtime(val)))
261+
except:
262+
mtime = "N/A"
263+
f.write("%s, %s, %s\n" % (val, size, mtime))
264+
sumSize += size
265+
duration = time.time() - start
266+
avgSpeed = (sumSize / duration) / 1000000.0
267+
progress = sumSize / float(totalSize) * 100.0
268+
print("Progress: %.2f%% Average Speed: %.3f MB/s" % (progress, avgSpeed))
269+
f.write("#Directories\n")
270+
for name in dirs:
271+
val = os.path.join(root, name)
272+
f.write("%s\n" % (val))
273+
else:
274+
raise NotImplementedError("Code for operating system of type %s is not implemented."%(os.name))
275+
276+
end = time.time()
277+
print("\nTook %.3f seconds.\n"%(end-start))
278+
279+
# Compress the backup file to save space
280+
zipped = zipfile.ZipFile(outputPath[:-4]+".zip", 'w')
281+
zipped.write(outputPath, compress_type=zipfile.ZIP_DEFLATED)
282+
zipped.close()
283+
284+
# Remove the original uncompressed backup file
285+
os.remove(outputPath)
286+
287+
288+
time.sleep(5)

0 commit comments

Comments
 (0)