Skip to content

Commit 80b4ed1

Browse files
andamianandamian
authored
vcp from vospace works when system in readonly mode (CADC 13277) (#218)
1 parent f66e357 commit 80b4ed1

File tree

11 files changed

+179
-207
lines changed

11 files changed

+179
-207
lines changed

‎README.rst‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ Note: might need to escape chars in your shell
3030
::
3131

3232
cd vos && pip install -e .[test]
33-
cd vofs && pip install -e .[test]
3433

3534
Testing packages
3635
----------------
@@ -43,14 +42,6 @@ Testing vos
4342
cd ./vos
4443
pytest vos
4544

46-
Testing vofs
47-
~~~~~~~~~~~~
48-
49-
::
50-
51-
cd ./vofs
52-
pytest vofs
53-
5445

5546

5647
Checkstyle
@@ -60,7 +51,7 @@ not report errors
6051

6152
::
6253

63-
flake8 vos/vos vofs/vofs
54+
flake8 vos/vos
6455

6556

6657
Testing with tox

‎vos/test/scripts/vospace-link-atest.tcsh‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -141,20 +141,8 @@ foreach resource ($resources)
141141
echo " [OK]"
142142
endif
143143

144-
echo -n "copy external link"
145-
if ( ${?TESTING_CAVERN} ) then
146-
echo " [SKIPPED, vos/issues/83]"
147-
else
148-
rm -f /tmp/e3link
149-
$CPCMD $CERT -L $CONTAINER/e3link /tmp/ >& /dev/null || echo " [FAIL]" && exit -1
150-
grep -q google /tmp/e3link || echo " [FAIL]" && exit -1
151-
rm -f /tmp/e3link
152-
echo " [OK]"
153-
endif
154-
155-
156144
echo -n "Follow the invalid link and fail"
157-
$CPCMD $CERT $CONTAINER/e3link/somefile /tmp >& /dev/null && echo " [FAIL]" && exit -1
145+
$CPCMD $CERT $CONTAINER/e3link/somefile /tmp >& /dev/null && echo " [FAIL]" && exit -
158146
echo " [OK]"
159147

160148
echo -n "copy file to target through link"

‎vos/test/scripts/vospace-lock-atest.tcsh‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -121,12 +121,10 @@ foreach resource ($resources)
121121
if ( ${?TESTING_CAVERN} ) then
122122
echo " [SKIPPED, vos/issues/82]"
123123
else
124-
echo "$LOCKCMD $CERT $CONTAINER $VUNLOCK_ARGS"
125124
$LOCKCMD $CERT $CONTAINER $VUNLOCK_ARGS > /dev/null || echo " [FAIL]" && exit -1
126125
echo " [OK]"
127126
echo -n "check unlocked container "
128-
echo "$TAGCMD $CERT $CONTAINER $LIST_ARGS"
129-
$TAGCMD $CERT $CONTAINER $LIST_ARGS | grep -q false && set SUCCESS = "true"
127+
$TAGCMD $CERT $CONTAINER $LIST_ARGS | grep -q None && set SUCCESS = "true"
130128

131129
if ( ${SUCCESS} == "true" ) then
132130
set SUCCESS = "false"
@@ -175,7 +173,7 @@ foreach resource ($resources)
175173
$LOCKCMD $CERT $CONTAINER/target $VUNLOCK_ARGS> /dev/null || echo " [FAIL]" && exit -1
176174
echo " [OK]"
177175
echo -n "check unlocked link "
178-
$TAGCMD $CERT $CONTAINER/target $LIST_ARGS | grep -q false && set SUCCESS = "true"
176+
$TAGCMD $CERT $CONTAINER/target $LIST_ARGS | grep -q None && set SUCCESS = "true"
179177

180178
if ( ${SUCCESS} == "true" ) then
181179
set SUCCESS = "false"
@@ -226,7 +224,7 @@ foreach resource ($resources)
226224
$LOCKCMD $CERT $CONTAINER/something.png $VUNLOCK_ARGS> /dev/null || echo " [FAIL]" && exit -1
227225
echo " [OK]"
228226
echo -n "check unlocked node "
229-
$TAGCMD $CERT $CONTAINER/something.png $LIST_ARGS | grep -q false && set SUCCESS = "true"
227+
$TAGCMD $CERT $CONTAINER/something.png $LIST_ARGS | grep -q None && set SUCCESS = "true"
230228
if ( ${SUCCESS} == "true" ) then
231229
set SUCCESS = "false"
232230
echo " [OK]"

‎vos/test/scripts/vospace-node-properties.tcsh‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ foreach resource ($resources)
5858
set TIMESTAMP=`date +%Y-%m-%dT%H-%M-%S`
5959
set CONTAINER = $BASE/$TIMESTAMP
6060

61-
echo -n "** checking base URI"
61+
# echo -n "** checking base URI"
6262
# $RMCMD -R $CERT $BASE > /dev/null
6363
# echo -n ", creating base URI"
6464
# $MKDIRCMD $CERT $BASE || echo " [FAIL]" && exit -1
@@ -107,24 +107,23 @@ foreach resource ($resources)
107107
$LSCMD $CERT $CONTAINER | grep ccc | grep -q "drw-------" || echo " [FAIL]" && exit -1
108108
echo " [OK]"
109109

110-
# echo -n "test vchmod with recursive option"
111-
#
112-
# $CHMODCMD $CERT -R g+r $CONTAINER $GROUP || echo " [FAIL]" && exit -1
113-
# echo -n " verify "
114-
# $LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1
115-
# $LSCMD $CERT $CONTAINER | grep aaa | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1
116-
# $LSCMD $CERT $CONTAINER | grep ccc | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1
117-
# $LSCMD $CERT $CONTAINER/aaa | grep bbb | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1
118-
# echo " [OK]"
110+
echo -n "test vchmod with recursive option"
111+
112+
$CHMODCMD $CERT -R g+r $CONTAINER $GROUP || echo " [FAIL]" && exit -1
113+
echo -n " verify "
114+
$LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1
115+
$LSCMD $CERT $CONTAINER | grep aaa | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1
116+
$LSCMD $CERT $CONTAINER | grep ccc | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1
117+
$LSCMD $CERT $CONTAINER/aaa | grep bbb | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1
118+
echo " [OK]"
119119

120120
echo -n "test vchmod with multiple groups"
121121

122122
set MULTIGROUP = "ABC $GROUP"
123-
$CHMODCMD $CERT g+r $CONTAINER/aaa "$MULTIGROUP" || echo " [FAIL]" && exit -1
123+
$CHMODCMD $CERT g+r $CONTAINER/aaa "$MULTIGROUP" || echo " [FAIL1]" && exit -1
124124
echo -n " verify "
125-
$LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL]" && exit -1
126-
echo ""
127-
$LSCMD $CERT $CONTAINER | grep aaa | grep "$MULTIGROUP" | grep -q "drw-r-----" || echo " [FAIL]" && exit -1
125+
$LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL2]" && exit -1
126+
$LSCMD $CERT $CONTAINER | grep aaa | grep "$MULTIGROUP" | grep -q "drw-r-----" || echo " [FAIL3]" && exit -1
128127
echo " [OK]"
129128

130129
echo -n "make a sub-container public"
@@ -134,7 +133,7 @@ foreach resource ($resources)
134133
$LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL1]" && exit -1
135134
$LSCMD $CERT $CONTAINER | grep aaa | grep "$MULTIGROUP" | grep -q "drw-r-----" || echo " [FAIL2]" && exit -1
136135
$LSCMD $CERT $CONTAINER | grep ccc | grep $GROUP | grep -q "drw-r-----" || echo " [FAIL3]" && exit -1
137-
$LSCMD $CERT $CONTAINER/aaa | grep bbb | grep "$MULTIGROUP" | grep -q "drw-r--r--" || echo " [FAIL4]" && exit -1
136+
$LSCMD $CERT $CONTAINER/aaa | grep bbb | grep "$GROUP" | grep -q "drw-r--r--" || echo " [FAIL4]" && exit -1
138137
echo " [OK]"
139138

140139
# echo -n "recursively make all directories public"

‎vos/test/scripts_old/vospace-client-atest.tcsh‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,9 @@ foreach resource ($resources)
7878
echo " [OK]"
7979
endif
8080
echo -n "** setting home and base to public, no groups"
81-
$CHMODCMD $CERT o+r $VOHOME || echo " [FAIL]" && exit -1
81+
$CHMODCMD $CERT o+r $VOHOME || echo " [FAIL1]" && exit -1
8282
echo -n " [OK]"
83-
$CHMODCMD $CERT o+r $BASE || echo " [FAIL]" && exit -1
83+
$CHMODCMD $CERT o+r $BASE || echo " [FAIL2]" && exit -1
8484
echo " [OK]"
8585
echo
8686
echo "*** starting test sequence ***"
@@ -187,14 +187,15 @@ foreach resource ($resources)
187187
echo " [OK]"
188188

189189
echo -n "copy data node to local filesystem "
190-
$CPCMD $CERT $CONTAINER/something.png $THIS_DIR/something.png.2 || echo " [FAIL]" && exit -1
191-
cmp $THIS_DIR/something.png $THIS_DIR/something.png.2 || echo " [FAIL]" && exit -1
190+
191+
$CPCMD $CERT $CONTAINER/something.png $THIS_DIR/something.png.2 || echo " [FAIL1]" && exit -1
192+
cmp $THIS_DIR/something.png $THIS_DIR/something.png.2 || echo " [FAIL2]" && exit -1
192193
\rm -f $THIS_DIR/something.png.2
193194
echo " [OK]"
194195

195196
echo -n "Check quick copy"
196-
$CPCMD $CERT --quick $THIS_DIR/something.png $CONTAINER/something.png.3|| echo " [FAIL]" && exit -1
197-
$CPCMD $CERT --quick $CONTAINER/something.png.3 $THIS_DIR/something.png.3 || echo " [FAIL]" && exit -1
197+
$CPCMD $CERT --quick $THIS_DIR/something.png $CONTAINER/something.png.3|| echo " [FAIL1]" && exit -1
198+
$CPCMD $CERT --quick $CONTAINER/something.png.3 $THIS_DIR/something.png.3 || echo " [FAIL2]" && exit -1
198199
cmp $THIS_DIR/something.png $THIS_DIR/something.png.3 || echo " [FAIL]" && exit -1
199200
\rm -f $THIS_DIR/something.png.3
200201
echo " [OK]"

‎vos/test/scripts_old/vospace-link-atest.tcsh‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -146,18 +146,6 @@ foreach resource ($resources)
146146
echo " [OK]"
147147
endif
148148

149-
echo -n "copy external link"
150-
if ( ${?TESTING_CAVERN} ) then
151-
echo " [SKIPPED, vos/issues/83]"
152-
else
153-
rm -f /tmp/e3link
154-
$CPCMD $CERT -L $CONTAINER/e3link /tmp/ >& /dev/null || echo " [FAIL]" && exit -1
155-
grep -q google /tmp/e3link || echo " [FAIL]" && exit -1
156-
rm -f /tmp/e3link
157-
echo " [OK]"
158-
endif
159-
160-
161149
echo -n "Follow the invalid link and fail"
162150
$CPCMD $CERT $CONTAINER/e3link/somefile /tmp >& /dev/null && echo " [FAIL]" && exit -1
163151
echo " [OK]"

‎vos/test/scripts_old/vospace-lock-atest.tcsh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ foreach resource ($resources)
133133
$LOCKCMD $CERT $CONTAINER $VUNLOCK_ARGS > /dev/null || echo " [FAIL]" && exit -1
134134
echo " [OK]"
135135
echo -n "check unlocked container "
136+
echo "$TAGCMD $CERT $CONTAINER $LIST_ARGS"
136137
$TAGCMD $CERT $CONTAINER $LIST_ARGS | grep -q None && set SUCCESS = "true"
137138

138139
if ( ${SUCCESS} == "true" ) then

‎vos/test/scripts_old/vospace-node-properties.tcsh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ foreach resource ($resources)
120120
echo " [FAIL]" && exit -1
121121
endif
122122
else
123+
echo "$CHMODCMD $CERT -R g+r $CONTAINER $GROUP1"
123124
$CHMODCMD $CERT -R g+r $CONTAINER $GROUP1 || echo " [FAIL]" && exit -1
124125
echo -n " verify "
125126
$LSCMD $CERT $BASE | grep $TIMESTAMP | grep $GROUP1 | grep -q "drw-r-----" || echo " [FAIL]" && exit -1

‎vos/vos/commands/vcp.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ class Nonlocal():
140140
help="DEPRECATED")
141141
parser.add_argument(
142142
"--quick", action="store_true",
143-
help="assuming CANFAR VOSpace, only comptible with CANFAR VOSpace.",
143+
help="DEPRECATED",
144144
default=False)
145145
parser.add_argument(
146146
"-L", "--follow-links",
@@ -169,7 +169,7 @@ class Nonlocal():
169169

170170
client = vos.Client(
171171
vospace_certfile=args.certfile, vospace_token=args.token,
172-
transfer_shortcut=args.quick, insecure=args.insecure)
172+
insecure=args.insecure)
173173

174174
if not client.is_remote_file(dest):
175175
dest = os.path.abspath(dest)

‎vos/vos/tests/test_vos.py‎

Lines changed: 32 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ def test_get_node_url():
122122

123123
response = Mock(spec=requests.Response)
124124
response.status_code = 303
125+
125126
resource_id = 'ivo://cadc.nrc.ca/vospace'
126127
session_mock = Mock(spec=requests.Session, get=Mock(return_value=response))
127128
session_mock.headers = Mock()
@@ -143,39 +144,28 @@ def test_get_node_url():
143144
order='desc')).query
144145
assert ('order=desc' == unquote(equery))
145146

146-
# test header view
147+
# test files URL
147148
transfer_url = 'https://mystorage.org/minoc/files/abc:VOS/002'
148-
client.transfer = Mock(return_value=[transfer_url])
149-
expected_url = transfer_url + '?META=true'
150-
assert expected_url == \
149+
response.headers = {'Location': transfer_url}
150+
mock_session = Mock(spec=requests.Session, get=Mock(return_value=response))
151+
client.get_session = Mock(return_value=mock_session)
152+
client._fs_type = False
153+
assert transfer_url == \
151154
client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
152-
view='header')[0]
155+
view='header')
156+
157+
# test fits header
158+
assert transfer_url + "?META=true" == client._add_soda_ops(transfer_url, view='header')
153159

154160
# test pixel cutouts
155-
transfer_url1 = 'https://mystorage.org/minoc/files/abc:VOS/001'
156-
transfer_url2 = 'https://myotherstorage.org/minoc/files/abc:VOS/001'
157-
client.transfer = Mock(return_value=[transfer_url1, transfer_url2])
158161
pcutout = '[1][100:125,100:175]'
159-
expected_url1 = transfer_url1 + '?SUB=' + pcutout
160-
expected_url2 = transfer_url2 + '?SUB=' + pcutout
161-
assert expected_url1 == \
162-
client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
163-
cutout=pcutout, view='cutout')[0]
164-
assert expected_url2 == \
165-
client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
166-
cutout=pcutout, view='cutout')[1]
162+
assert transfer_url + "?SUB=" + pcutout == client._add_soda_ops(transfer_url,
163+
cutout=pcutout)
167164

168165
# test sky coordinates
169-
client.transfer = Mock(return_value=[transfer_url1, transfer_url2])
170-
scutout = 'CIRCLE=' + urllib.parse.quote('(1.1 2.2 3.3')
171-
expected_url1 = transfer_url1 + '?' + scutout
172-
expected_url2 = transfer_url2 + '?' + scutout
173-
assert expected_url1 == \
174-
client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
175-
cutout=scutout, view='cutout')[0]
176-
assert expected_url2 == \
177-
client.get_node_url('vos://cadc.nrc.ca!vospace/auser',
178-
cutout=scutout, view='cutout')[1]
166+
scutout = '1.1,2.2,3.3'
167+
assert (transfer_url + "?CIRCLE=" + urllib.parse.quote(scutout) ==
168+
client._add_soda_ops(transfer_url, cutout='CIRCLE='+scutout))
179169

180170

181171
class TestClient(unittest.TestCase):
@@ -444,6 +434,7 @@ def is_remote_file(uri):
444434
os.remove(osLocation)
445435
# copy from vospace
446436
test_client.is_remote_file = is_remote_file
437+
test_client.get_endpoints = Mock()
447438
test_client.copy(vospaceLocation, osLocation)
448439
get_node_url_mock.assert_called_once_with(vospaceLocation,
449440
method='GET',
@@ -488,11 +479,21 @@ def is_remote_file(uri):
488479
response.iter_content.return_value = BytesIO(file_content)
489480
session.get.return_value = response
490481
test_client.get_session = Mock(return_value=session)
482+
# client must be a vault client
483+
test_client._fs_type = False
491484
test_client.copy('{}{}'.format(vospaceLocation,
492485
'[1][10:60]'), osLocation)
493486
get_node_url_mock.assert_called_once_with(
494487
vospaceLocation, method='GET', cutout='[1][10:60]', view='cutout')
495488

489+
# test cavern does not support SODA operations
490+
test_client._fs_type = True
491+
with pytest.raises(ValueError):
492+
test_client.copy('{}{}'.format(vospaceLocation, '[1][10:60]'), osLocation)
493+
with pytest.raises(ValueError):
494+
test_client.copy(vospaceLocation, osLocation, head=True)
495+
496+
test_client._fs_type = False
496497
# copy to vospace when md5 sums are the same -> only update occurs
497498
get_node_url_mock.reset_mock()
498499
computed_md5_mock.reset_mock()
@@ -603,8 +604,8 @@ def is_remote_file(uri):
603604

604605
# test GET intermittent exceptions on both URLs
605606
props.get.side_effect = md5sum
606-
get_node_url_mock = Mock(
607-
return_value=['http://cadc1.ca/test', 'http://cadc2.ca/test'])
607+
# first side effect corresponds to the files end point call, the second to full negotiation
608+
get_node_url_mock = Mock(side_effect=['http://cadc1.ca/test', ['http://cadc2.ca/test']])
608609
test_client.get_node_url = get_node_url_mock
609610
get_node_mock.reset_mock()
610611
response.iter_content.return_value = BytesIO(file_content)
@@ -613,13 +614,13 @@ def is_remote_file(uri):
613614
session.get.side_effect = \
614615
[exceptions.TransferException()] * 2 * vos.MAX_INTERMTTENT_RETRIES
615616
with pytest.raises(OSError):
616-
test_client.copy(vospaceLocation, osLocation, head=True)
617+
test_client.copy(vospaceLocation, osLocation, head=False)
617618
assert session.get.call_count == 2 * vos.MAX_INTERMTTENT_RETRIES
618619

619620
# test GET Transfer error on one URL and a "permanent" one on the other
620621
props.get.side_effect = md5sum
621622
get_node_url_mock = Mock(
622-
return_value=['http://cadc1.ca/test', 'http://cadc2.ca/test'])
623+
side_effect=[None, ['http://cadc1.ca/test', 'http://cadc2.ca/test']])
623624
test_client.get_node_url = get_node_url_mock
624625
get_node_mock.reset_mock()
625626
response.iter_content.return_value = BytesIO(file_content)
@@ -636,7 +637,7 @@ def is_remote_file(uri):
636637
# test GET both "permanent" errors
637638
props.get.side_effect = md5sum
638639
get_node_url_mock = Mock(
639-
return_value=['http://cadc1.ca/test', 'http://cadc2.ca/test'])
640+
side_effect=['http://cadc1.ca/test', ['http://cadc2.ca/test']])
640641
test_client.get_node_url = get_node_url_mock
641642
get_node_mock.reset_mock()
642643
response.iter_content.return_value = BytesIO(file_content)

0 commit comments

Comments
 (0)