Skip to content

Commit 0580e1c

Browse files
tw4ltw4l
authored
Merge pull request #9 from tw4l/master
SIP Creator v1.1.0
2 parents 5e9ac8e + 5665c1b commit 0580e1c

File tree

11 files changed

+341
-51
lines changed

11 files changed

+341
-51
lines changed

‎.github/workflows/test.yml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
---
2+
name: "Test"
3+
on:
4+
pull_request:
5+
push:
6+
7+
env:
8+
ACTIONS_ALLOW_UNSECURE_COMMANDS: True
9+
10+
jobs:
11+
tox:
12+
name: "Test ${{ matrix.toxenv }}"
13+
runs-on: "ubuntu-18.04"
14+
strategy:
15+
matrix:
16+
include:
17+
- python-version: "3.7"
18+
toxenv: "py37"
19+
- python-version: "3.8"
20+
toxenv: "py38"
21+
- python-version: "3.9"
22+
toxenv: "py39"
23+
- python-version: "3.10"
24+
toxenv: "py310"
25+
steps:
26+
- uses: actions/checkout@v2
27+
with:
28+
submodules: "true"
29+
- name: "Set up Python ${{ matrix.python-version }}"
30+
uses: "actions/setup-python@v2"
31+
with:
32+
python-version: "${{ matrix.python-version }}"
33+
- name: Install homebrew
34+
run: |
35+
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
36+
test -d ~/.linuxbrew && eval $(~/.linuxbrew/bin/brew shellenv)
37+
test -d /home/linuxbrew/.linuxbrew && eval $(/home/linuxbrew/.linuxbrew/bin/brew shellenv)
38+
echo "eval \$($(brew --prefix)/bin/brew shellenv)" >>~/.profile
39+
echo "::add-path::/home/linuxbrew/.linuxbrew/bin"
40+
brew --version
41+
- name: Install Siegfried
42+
run: |
43+
brew install richardlehane/digipres/siegfried
44+
- name: Install md5deep
45+
run: |
46+
brew install md5deep
47+
- name: Install bulk_extractor
48+
run: |
49+
brew install bulk_extractor
50+
- name: Run install script
51+
run: |
52+
./test-install.sh
53+
shell: bash
54+
- name: "Get pip cache dir"
55+
id: "pip-cache"
56+
run: |
57+
echo "::set-output name=dir::$(pip cache dir)"
58+
- name: "Cache pip packages"
59+
uses: "actions/cache@v2"
60+
with:
61+
path: "${{ steps.pip-cache.outputs.dir }}"
62+
key: "${{ runner.os }}-pip-${{ hashFiles('**/base.txt','**/test.txt') }}"
63+
restore-keys: |
64+
${{ runner.os }}-pip-
65+
- name: "Install tox"
66+
run: |
67+
python -m pip install --upgrade pip
68+
pip install tox
69+
- name: "Run tox"
70+
env:
71+
TOXENV: ${{ matrix.toxenv }}
72+
run: |
73+
tox

‎README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# SIP Creator
22

3-
Version: 1.0.0
3+
Version: 1.1.0
44

55
Creates an Archivematica-ready transfer (or SIP for another preservation repository) from user-selected directories and files, and generates a pre-populated description spreadsheet using data pulled from DFXML and Brunnhilde (start and end dates, extent, and a scope and content note containing information about the most common file formats present).
66

@@ -16,12 +16,12 @@ This utility is designed for easy use in BitCurator v1.8.0+. It requires Python
1616

1717
Install all of the CCA Tools together using the installation script in the [CCA Tools repo](https://github.com/CCA-Public/cca-tools).
1818

19-
### Install as a separate utlity
19+
### Install as a separate utility
20+
2021
* Install [PyQt5](https://www.riverbankcomputing.com/software/pyqt/download5):
2122
`sudo pip3 install pyqt5`
2223
* Clone this repo to your local machine.
23-
* Run the install script with sudo privileges:
24-
`sudo chmod u+x install.sh`
24+
* Run the install script with sudo privileges (assuming BitCurator 4; for BitCurator 2-3 run `./install-bc2-ubuntu18.sh` instead):
2525
`sudo ./install.sh`
2626

2727
### PyQt4 version

‎install-bc2-ubuntu18.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
3+
### Install script for CCA SIP Creator in Bitcurator
4+
5+
# Update submodules
6+
git submodule update --init --recursive
7+
8+
# Make /usr/share/ccatools if doesn't already exist
9+
if [ ! -d /usr/share/ccatools ]; then
10+
sudo mkdir /usr/share/ccatools
11+
fi
12+
13+
# Delete /usr/share directory for SIP Creator if it already exists
14+
if [ -d /usr/share/ccatools/sipcreator ]; then
15+
sudo rm -rf /usr/share/ccatools/sipcreator
16+
fi
17+
18+
# Make /usr/share directory for SIP Creator
19+
sudo mkdir /usr/share/ccatools/sipcreator
20+
21+
# Move files into /usr/share/ccatools/sipcreator
22+
sudo cp main.py /usr/share/ccatools/sipcreator
23+
sudo cp launch /usr/share/ccatools/sipcreator
24+
sudo cp design.py /usr/share/ccatools/sipcreator
25+
sudo cp design.ui /usr/share/ccatools/sipcreator
26+
sudo cp icon.png /usr/share/ccatools/sipcreator
27+
sudo cp LICENSE /usr/share/ccatools/sipcreator
28+
sudo cp README.md /usr/share/ccatools/sipcreator
29+
sudo cp deps/dfxml/python/dfxml.py /usr/share/ccatools/sipcreator
30+
sudo cp deps/dfxml/python/Objects.py /usr/share/ccatools/sipcreator
31+
sudo cp deps/dfxml/python/walk_to_dfxml.py /usr/share/ccatools/sipcreator
32+
33+
# Make "CCA Tools" folder on Desktop if doesn't already exist
34+
if [ ! -d "/home/bcadmin/Desktop/CCA Tools" ]; then
35+
sudo mkdir "/home/bcadmin/Desktop/CCA Tools"
36+
fi
37+
38+
# Create launch.desktop file
39+
sudo touch '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
40+
echo '[Desktop Entry]' | sudo tee --append '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
41+
echo 'Type=Application' | sudo tee --append '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
42+
echo 'Name=SIP Creator' | sudo tee --append '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
43+
echo 'Exec=/usr/share/ccatools/sipcreator/launch' | sudo tee --append '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
44+
echo 'Icon=/usr/share/ccatools/sipcreator/icon.png' | sudo tee --append '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
45+
46+
# Change permissions, ownership for CCA Tools
47+
sudo chown -R bcadmin:bcadmin '/home/bcadmin/Desktop/CCA Tools'
48+
sudo chown -R bcadmin:bcadmin '/usr/share/ccatools/sipcreator'
49+
sudo find '/home/bcadmin/Desktop/CCA Tools' -type d -exec chmod 755 {} \;
50+
sudo find '/home/bcadmin/Desktop/CCA Tools' -type f -exec chmod 644 {} \;
51+
52+
# Make files executable
53+
sudo chmod u+x '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
54+
sudo chmod u+x /usr/share/ccatools/sipcreator/launch

‎install.sh‎

100755100644
Lines changed: 32 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,46 @@
11
#!/bin/bash
22

3-
### Install script for CCA SIP Creator in Bitcurator
3+
### Install script for CCA Disk Image Processor in Bitcurator 4/Ubuntu 22
44

5-
# Update submodules
65
git submodule update --init --recursive
76

8-
# Make /usr/share/ccatools if doesn't already exist
97
if [ ! -d /usr/share/ccatools ]; then
108
sudo mkdir /usr/share/ccatools
119
fi
1210

13-
# Delete /usr/share directory for SIP Creator if it already exists
14-
if [ -d /usr/share/ccatools/sipcreator ]; then
15-
sudo rm -rf /usr/share/ccatools/sipcreator
16-
fi
11+
sipcreator_dir="/usr/share/ccatools/sipcreator/"
1712

18-
# Make /usr/share directory for SIP Creator
19-
sudo mkdir /usr/share/ccatools/sipcreator
20-
21-
# Move files into /usr/share/ccatools/sipcreator
22-
sudo cp main.py /usr/share/ccatools/sipcreator
23-
sudo cp launch /usr/share/ccatools/sipcreator
24-
sudo cp design.py /usr/share/ccatools/sipcreator
25-
sudo cp design.ui /usr/share/ccatools/sipcreator
26-
sudo cp icon.png /usr/share/ccatools/sipcreator
27-
sudo cp LICENSE /usr/share/ccatools/sipcreator
28-
sudo cp README.md /usr/share/ccatools/sipcreator
29-
sudo cp deps/dfxml/python/dfxml.py /usr/share/ccatools/sipcreator
30-
sudo cp deps/dfxml/python/Objects.py /usr/share/ccatools/sipcreator
31-
sudo cp deps/dfxml/python/walk_to_dfxml.py /usr/share/ccatools/sipcreator
32-
33-
# Make "CCA Tools" folder on Desktop if doesn't already exist
34-
if [ ! -d "/home/bcadmin/Desktop/CCA Tools" ]; then
35-
sudo mkdir "/home/bcadmin/Desktop/CCA Tools"
13+
if [ -d $sipcreator_dir ]; then
14+
sudo rm -rf $sipcreator_dir
3615
fi
3716

17+
sudo mkdir $sipcreator_dir
18+
19+
sudo cp main.py $sipcreator_dir
20+
sudo cp launch $sipcreator_dir
21+
sudo cp design.py $sipcreator_dir
22+
sudo cp design.ui $sipcreator_dir
23+
sudo cp icon.png $sipcreator_dir
24+
sudo cp LICENSE $sipcreator_dir
25+
sudo cp README.md $sipcreator_dir
26+
sudo cp deps/dfxml/python/dfxml.py $sipcreator_dir
27+
sudo cp deps/dfxml/python/Objects.py $sipcreator_dir
28+
sudo cp deps/dfxml/python/walk_to_dfxml.py $sipcreator_dir
29+
3830
# Create launch.desktop file
39-
sudo touch '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
40-
echo '[Desktop Entry]' | sudo tee --append '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
41-
echo 'Type=Application' | sudo tee --append '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
42-
echo 'Name=SIP Creator' | sudo tee --append '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
43-
echo 'Exec=/usr/share/ccatools/sipcreator/launch' | sudo tee --append '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
44-
echo 'Icon=/usr/share/ccatools/sipcreator/icon.png' | sudo tee --append '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
45-
46-
# Change permissions, ownership for CCA Tools
47-
sudo chown -R bcadmin:bcadmin '/home/bcadmin/Desktop/CCA Tools'
48-
sudo chown -R bcadmin:bcadmin '/usr/share/ccatools/sipcreator'
49-
sudo find '/home/bcadmin/Desktop/CCA Tools' -type d -exec chmod 755 {} \;
50-
sudo find '/home/bcadmin/Desktop/CCA Tools' -type f -exec chmod 644 {} \;
51-
52-
# Make files executable
53-
sudo chmod u+x '/home/bcadmin/Desktop/CCA Tools/SIP Creator.desktop'
31+
launch_file="/usr/share/applications/SIPCreator.desktop"
32+
33+
if [ -f $launch_file ]; then
34+
sudo rm -rf $launch_file
35+
fi
36+
37+
sudo touch $launch_file
38+
echo '[Desktop Entry]' | sudo tee --append $launch_file
39+
echo 'Type=Application' | sudo tee --append $launch_file
40+
echo 'Name=SIP Creator' | sudo tee --append $launch_file
41+
echo 'Exec=/usr/share/ccatools/sipcreator/launch' | sudo tee --append $launch_file
42+
echo 'Icon=/usr/share/ccatools/sipcreator/icon.png' | sudo tee --append $launch_file
43+
echo 'Categories=Forensics and Reporting' | sudo tee --append $launch_file
44+
45+
sudo chown -R bcadmin:bcadmin $sipcreator_dir
5446
sudo chmod u+x /usr/share/ccatools/sipcreator/launch

‎main.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
44
(c) Canadian Centre for Architecture
55
Developed by Tessa Walsh
6-
2017-2021
6+
2017-2023
77
MIT License
88
"""
99
import csv
@@ -63,7 +63,7 @@ def convert_size(size):
6363
s = round(size / p)
6464
s = str(s)
6565
s = s.replace(".0", "")
66-
return "%s %s" % (s, size_name[i])
66+
return "{} {}".format(s, size_name[i])
6767

6868

6969
class CheckableDirModel(QDirModel):
@@ -162,7 +162,10 @@ def create_sip(self, files_to_process, sip_dir, bag_files, scan_for_pii):
162162

163163
# Bag files or write checksum manifest.
164164
if bag_files:
165-
subprocess.call("bagit.py --processes 4 '{}'".format(sip_dir), shell=True)
165+
# TODO: Multithread bagging via --processes when bug described at
166+
# https://github.com/LibraryOfCongress/bagit-python/issues/130 is
167+
# resolved.
168+
subprocess.call("cd ~ && bagit.py '{}'".format(sip_dir), shell=True)
166169
else:
167170
md5deep_cmd = "cd '{}' && md5deep -rl ../objects > checksum.md5".format(
168171
metadata_dir
@@ -226,8 +229,8 @@ def write_csv_row(writer, sip_path, bag_files):
226229
if mtimes:
227230
date_earliest = min(mtimes)[:10]
228231
date_latest = max(mtimes)[:10]
229-
date_statement = "{} - {}".format(date_earliest[:4], date_latest[:4])
230-
if date_earliest == date_latest:
232+
date_statement = "{}-{}".format(date_earliest[:4], date_latest[:4])
233+
if date_earliest[:4] == date_latest[:4]:
231234
date_statement = date_earliest[:4]
232235

233236
# Write scope and content note from information in brunnhilde reports.
@@ -259,7 +262,7 @@ def write_csv_row(writer, sip_path, bag_files):
259262
file_formats.append(row[0])
260263
file_formats = [format_ or "Unidentified" for format_ in file_formats]
261264
formats_list = ", ".join(file_formats)
262-
scope_content = 'Original directory name: "{}". Most common file formats: {}'.format(
265+
scope_content = "Most common file formats: {}".format(
263266
os.path.basename(sip_path), formats_list
264267
)
265268

@@ -325,7 +328,7 @@ def about_dialog(self):
325328
QMessageBox.information(
326329
self,
327330
"About",
328-
"SIP Creator v1.0.0\nCanadian Centre for Architecture\nDeveloper: Tessa Walsh\n2018-2021\nMIT License\nhttps://github.com/CCA-Public/sipcreator",
331+
"SIP Creator v1.1.0\nCanadian Centre for Architecture\nDeveloper: Tessa Walsh\n2018-2023\nMIT License\nhttps://github.com/CCA-Public/sipcreator",
329332
)
330333

331334
def browse_source(self):

‎requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
-r requirements/base.txt

‎requirements/base.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
bagit
2+
brunnhilde
3+
pyqt5

‎requirements/test.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
-r base.txt
2+
3+
pytest==6.2.4
4+
pytest-mock==3.6.1
5+
tox==3.23.1

‎test-install.sh‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/bin/bash
2+
3+
### Install SIP Creator for testing
4+
5+
# Update submodules
6+
git submodule update --init --recursive
7+
8+
# Make /usr/share/ccatools if doesn't already exist
9+
if [ ! -d /usr/share/ccatools ]; then
10+
sudo mkdir /usr/share/ccatools
11+
fi
12+
13+
# Delete /usr/share directory for SIP Creator if it already exists
14+
if [ -d /usr/share/ccatools/sipcreator ]; then
15+
sudo rm -rf /usr/share/ccatools/sipcreator
16+
fi
17+
18+
# Make /usr/share directory for SIP Creator
19+
sudo mkdir /usr/share/ccatools/sipcreator
20+
21+
# Move files into /usr/share/ccatools/sipcreator
22+
sudo cp main.py /usr/share/ccatools/sipcreator
23+
sudo cp launch /usr/share/ccatools/sipcreator
24+
sudo cp design.py /usr/share/ccatools/sipcreator
25+
sudo cp design.ui /usr/share/ccatools/sipcreator
26+
sudo cp icon.png /usr/share/ccatools/sipcreator
27+
sudo cp LICENSE /usr/share/ccatools/sipcreator
28+
sudo cp README.md /usr/share/ccatools/sipcreator
29+
sudo cp deps/dfxml/python/dfxml.py /usr/share/ccatools/sipcreator
30+
sudo cp deps/dfxml/python/Objects.py /usr/share/ccatools/sipcreator
31+
sudo cp deps/dfxml/python/walk_to_dfxml.py /usr/share/ccatools/sipcreator
32+
33+
sudo cp deps/dfxml/python/dfxml.py .
34+
sudo cp deps/dfxml/python/Objects.py .
35+
sudo cp deps/dfxml/python/walk_to_dfxml.py .

0 commit comments

Comments
 (0)