Skip to content

Commit 34d40a7

Browse files
jnmjnm
authored
Merge pull request #238 from kobotoolbox/236-library-locking
Library locking
2 parents 972fde2 + f7ba636 commit 34d40a7

File tree

8 files changed

+589
-6
lines changed

8 files changed

+589
-6
lines changed

‎dev-requirements-py3.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,4 @@ funcsigs==1.0.2
1414
pytest-cov==2.6.1
1515
coveralls==1.5.1
1616
geojson-rewind==0.2.0
17+
xlwt==1.3.0

‎src/formpack/constants.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,35 @@
115115
TRUE,
116116
]
117117

118+
# Kobo locking constants
119+
KOBO_LOCK_ALL = 'kobo--lock_all'
120+
KOBO_LOCK_COLUMN = 'kobo--locking-profile'
121+
KOBO_LOCK_KEY = 'locked'
122+
KOBO_LOCK_SHEET = 'kobo--locking-profiles'
123+
KOBO_LOCKING_RESTRICTIONS = [
124+
'choice_add',
125+
'choice_delete',
126+
'choice_label_edit',
127+
'choice_value_edit',
128+
'choice_order_edit',
129+
'question_delete',
130+
'question_label_edit',
131+
'question_settings_edit',
132+
'question_skip_logic_edit',
133+
'question_validation_edit',
134+
'group_delete',
135+
'group_label_edit',
136+
'group_question_add',
137+
'group_question_delete',
138+
'group_question_order_edit',
139+
'group_settings_edit',
140+
'group_skip_logic_edit',
141+
'group_split',
142+
'form_replace',
143+
'group_add',
144+
'question_add',
145+
'question_order_edit',
146+
'language_edit',
147+
'form_appearance',
148+
'form_meta_edit',
149+
]

‎src/formpack/utils/exceptions.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,7 @@
33

44
class FormPackGeoJsonError(Exception):
55
pass
6+
7+
8+
class FormPackLibraryLockingError(Exception):
9+
pass

‎src/formpack/utils/kobo_locking.py‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
# coding: utf-8
2+
from __future__ import (unicode_literals, print_function,
3+
absolute_import, division)
4+
5+
import io
6+
import itertools
7+
from collections import OrderedDict
8+
9+
from .xls_to_ss_structure import xls_to_dicts
10+
from formpack.constants import (
11+
KOBO_LOCKING_RESTRICTIONS,
12+
KOBO_LOCK_COLUMN,
13+
KOBO_LOCK_KEY,
14+
KOBO_LOCK_SHEET,
15+
)
16+
from formpack.utils.exceptions import FormPackLibraryLockingError
17+
18+
def get_kobo_locking_profiles(xls_file_object: io.BytesIO) -> list:
19+
"""
20+
Return the locking profiles, if there are any, in a dictionary structure
21+
from an XLSForm matrix. For example, the following matrix structure:
22+
23+
# kobo--locking-profiles
24+
| restriction | profile_1 | profile_2 |
25+
|-------------------|-----------|-----------|
26+
| choice_add | locked | |
27+
| choice_delete | | locked |
28+
| choice_label_edit | locked | |
29+
| choice_order_edit | locked | locked |
30+
31+
Will be transformed into the following JSON structure:
32+
[
33+
{
34+
"name": "profile_1",
35+
"restrictions": [
36+
"choice_add",
37+
"choice_label_edit",
38+
"choice_order_edit"
39+
],
40+
},
41+
{
42+
"name": "profile_2",
43+
"restrictions": [
44+
"choice_delete",
45+
"choice_order_edit"
46+
],
47+
}
48+
]
49+
"""
50+
survey_dict = xls_to_dicts(xls_file_object)
51+
52+
if KOBO_LOCK_SHEET not in survey_dict:
53+
return
54+
55+
locks = survey_dict[KOBO_LOCK_SHEET]
56+
57+
# Get a unique list of profile names
58+
profiles = set()
59+
for lock in locks:
60+
profiles.update(lock.keys())
61+
62+
# So some basic validation of locking profiles
63+
profiles = _validate_locking_profiles(profiles)
64+
65+
# Set up an indexed dictionary for convenience -- return only its values
66+
locking_profiles = {
67+
name: dict(name=name, restrictions=[]) for name in profiles
68+
}
69+
70+
for lock in locks:
71+
restriction = lock.get('restriction')
72+
# ensure that valid lock values are being used
73+
if restriction not in KOBO_LOCKING_RESTRICTIONS:
74+
raise FormPackLibraryLockingError(
75+
f'{restriction} is not a valid restriction.'
76+
)
77+
for name in profiles:
78+
if lock.get(name, '').lower() == KOBO_LOCK_KEY:
79+
locking_profiles[name]['restrictions'].append(restriction)
80+
81+
return list(locking_profiles.values())
82+
83+
def revert_kobo_lock_structure(content: dict) -> None:
84+
"""
85+
Revert the structure of the locks to one that is ready to be exported into
86+
an XLSForm again -- the reverse of `get_kobo_locking_profiles`
87+
88+
It is essentially a preprocessor used within KPI before converting all the
89+
sheets and content to OrderedDicts and exporting to XLS.
90+
91+
For example, this JSON structure:
92+
[
93+
{
94+
"name": "profile_1",
95+
"restrictions": [
96+
"choice_add",
97+
"choice_label_edit",
98+
"choice_order_edit"
99+
],
100+
},
101+
{
102+
"name": "profile_2",
103+
"restrictions": [
104+
"choice_delete",
105+
"choice_order_edit"
106+
],
107+
}
108+
]
109+
110+
Will be transformed into:
111+
[
112+
{
113+
'restriction': 'choice_add',
114+
'profile_1': 'locked',
115+
},
116+
{
117+
'restriction': 'choice_label_edit',
118+
'profile_1': 'locked',
119+
},
120+
{
121+
'restriction': 'choice_order_edit',
122+
'profile_1': 'locked',
123+
'profile_2': 'locked',
124+
},
125+
{
126+
'restriction': 'choice_delete',
127+
'profile_2': 'locked',
128+
},
129+
]
130+
"""
131+
if KOBO_LOCK_SHEET not in content:
132+
return
133+
locking_profiles = []
134+
for res in KOBO_LOCKING_RESTRICTIONS:
135+
profile = {'restriction': res}
136+
for item in content[KOBO_LOCK_SHEET]:
137+
name = item['name']
138+
restrictions = item['restrictions']
139+
if res in restrictions:
140+
profile[name] = KOBO_LOCK_KEY
141+
locking_profiles.append(profile)
142+
content[KOBO_LOCK_SHEET] = locking_profiles
143+
144+
def strip_kobo_locking_profile(content: OrderedDict) -> None:
145+
"""
146+
Strip all `kobo--locking-profile` values from a survey. Used when creating
147+
blocks or adding questions to the library from a locked survey or template.
148+
The locks should only be applied on survey and template types.
149+
"""
150+
survey = content.get('survey')
151+
for item in survey:
152+
if KOBO_LOCK_COLUMN in item:
153+
item.pop(KOBO_LOCK_COLUMN)
154+
155+
def _validate_locking_profiles(profiles):
156+
"""
157+
Some simple validation of the locking profiles to provide helpful error
158+
messages to the user
159+
"""
160+
if 'restriction' not in profiles:
161+
raise FormPackLibraryLockingError(
162+
'The column name `restriction` must be present.'
163+
)
164+
165+
# Remove the `restriction` column header from the list to only have the
166+
# user-defined profile names
167+
profiles.remove('restriction')
168+
169+
if not profiles:
170+
raise FormPackLibraryLockingError(
171+
'At least one locking profile must be defined.'
172+
)
173+
174+
if KOBO_LOCK_KEY in profiles:
175+
raise FormPackLibraryLockingError(
176+
f'Locking profile name of "{KOBO_LOCK_KEY}" cannot be used.'
177+
)
178+
179+
return profiles

‎src/formpack/utils/replace_aliases.py‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,14 @@
55
from collections import defaultdict
66
from copy import deepcopy
77
import json
8+
import re
89

910
from pyxform import aliases as pyxform_aliases
1011
from pyxform.question_type_dictionary import QUESTION_TYPE_DICT
1112

1213
from .future import iteritems, OrderedDict
1314
from .string import str_types
15+
from ..constants import KOBO_LOCK_ALL
1416

1517
# This file is a mishmash of things which culminate in the
1618
# "replace_aliases" method which iterates through a survey and
@@ -22,6 +24,8 @@
2224
'required',
2325
]
2426

27+
KOBO_SPECIFIC_SUB_PATTERN = r'^kobo(–|—)'
28+
KOBO_SPECIFIC_PREFERRED = 'kobo--'
2529

2630
def aliases_to_ordered_dict(_d):
2731
"""
@@ -221,6 +225,17 @@ def _unpack_headers(p_aliases, fp_preferred):
221225
formpack_preferred_survey_headers)
222226

223227

228+
def kobo_specific_sub(key: str) -> str:
229+
"""
230+
Ensure that kobo-specific names (kobo--*) that happen to start with n-dash
231+
or m-dash characters are substituted with two single dashes for
232+
consistency. This accommodates for some software that will automatically
233+
substitute two dashes for a single n-dash or m-dash character. For example:
234+
`kobo–something` -> `kobo--something`,
235+
`kobo—something` -> `kobo--soemthing`
236+
"""
237+
return re.sub(KOBO_SPECIFIC_SUB_PATTERN, KOBO_SPECIFIC_PREFERRED, key)
238+
224239
def dealias_type(type_str, strict=False, allowed_types=None):
225240
if allowed_types is None:
226241
allowed_types = {}
@@ -266,6 +281,12 @@ def replace_aliases_in_place(content, allowed_types=None):
266281
row[val] = row[key]
267282
del row[key]
268283

284+
for key, val in row.copy().items():
285+
if re.search(KOBO_SPECIFIC_SUB_PATTERN, key) is not None:
286+
new_key = kobo_specific_sub(key)
287+
row[new_key] = val
288+
del row[key]
289+
269290
for row in content.get('choices', []):
270291
if 'list name' in row:
271292
row['list_name'] = row.pop('list name')
@@ -281,7 +302,13 @@ def replace_aliases_in_place(content, allowed_types=None):
281302
' first been parsed through "expand_content".')
282303

283304
if settings:
284-
content['settings'] = dict([
285-
(settings_header_columns.get(key, key), val)
286-
for key, val in settings.items()
287-
])
305+
_settings = {}
306+
for key, val in settings.items():
307+
_key = kobo_specific_sub(settings_header_columns.get(key, key))
308+
_val = (
309+
pyxform_aliases.yes_no.get(val, val)
310+
if _key == KOBO_LOCK_ALL
311+
else val
312+
)
313+
_settings[_key] = _val
314+
content['settings'] = _settings

‎src/formpack/utils/xls_to_ss_structure.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from .future import OrderedDict, unichr
1010
from .string import unicode, str_types
1111

12+
from .replace_aliases import kobo_specific_sub
1213

1314
def xls_to_lists(xls_file_object, strip_empty_rows=True):
1415
"""
@@ -80,7 +81,7 @@ def _sheet_to_lists(sheet):
8081
workbook = xlrd.open_workbook(file_contents=xls_file_object.read())
8182
ss_structure = OrderedDict()
8283
for sheet in workbook.sheets():
83-
sheet_name = sheet.name
84+
sheet_name = kobo_specific_sub(sheet.name)
8485
sheet_contents = _sheet_to_lists(sheet)
8586
ss_structure[sheet_name] = sheet_contents
8687
return ss_structure

0 commit comments

Comments
 (0)