Skip to content

Commit 40c1767

Browse files
andamianandamian
authored
Changes to make vostools work with the new cavern and vault services (CADC-12912) (#215)
1 parent d4edf89 commit 40c1767

File tree

12 files changed

+301
-162
lines changed

12 files changed

+301
-162
lines changed

‎.github/workflows/cibuild.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ jobs:
2828
strategy:
2929
fail-fast: true
3030
matrix:
31-
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
31+
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
3232
package: [vos]
3333
steps:
3434
- name: Checkout code

‎vos/setup.cfg‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@ omit = */tests/*
2121
[tool:pytest]
2222
minversion = 2.2
2323
norecursedirs = build docs/_build
24-
doctest_plus = enabled
2524
testpaths = vos
2625

26+
[flake8]
27+
max-line-length = 120
28+
2729
[pep8]
2830
# E101 - mix of tabs and spaces
2931
# W191 - use of tabs
@@ -54,7 +56,7 @@ install_requires =
5456
aenum
5557

5658
# version should be PEP440 compatible (http://www.python.org/dev/peps/pep-0440)
57-
version = 3.4
59+
version = 3.5
5860

5961
[options.extras_require]
6062
test =

‎vos/setup.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import glob
55
import os
66
import sys
7-
import imp
87
from setuptools.command.test import test as TestCommand
98
from setuptools import find_packages
109

@@ -128,7 +127,7 @@ def run_tests(self):
128127
use_2to3=False,
129128
setup_requires=['pytest-runner'],
130129
entry_points=entry_points,
131-
python_requires='!=3.0.*, !=3.1.*, !=3.2.*, !=3.3, <4',
130+
python_requires='>=3.7, <4',
132131
packages=find_packages(),
133132
package_data={PACKAGENAME: ['data/*', 'tests/data/*', '*/data/*', '*/tests/data/*']},
134133
classifiers=[

‎vos/tox.ini‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ name = vos
55

66
[tox]
77
envlist =
8-
py{37,38,39,310,311}
8+
py{37,38,39,310,311,312}
99
requires =
1010
pip >= 19.3.1
1111

‎vos/vos/commands/vchmod.py‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# ****************** CANADIAN ASTRONOMY DATA CENTRE *******************
33
# ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
44
#
5-
# (c) 2022. (c) 2022.
5+
# (c) 2024. (c) 2024.
66
# Government of Canada Gouvernement du Canada
77
# National Research Council Conseil national de recherches
88
# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
@@ -71,7 +71,7 @@
7171
"""
7272
from ..vos import Client
7373
from ..vos import CADC_GMS_PREFIX
74-
from ..commonparser import CommonParser, set_logging_level_from_args,\
74+
from ..commonparser import CommonParser, set_logging_level_from_args, \
7575
URI_DESCRIPTION
7676
from ..commonparser import exit_on_exception
7777
import logging
@@ -190,20 +190,23 @@ def vchmod():
190190
vospace_token=opt.token,
191191
insecure=opt.insecure)
192192
node = client.get_node(opt.node)
193-
if opt.recursive:
194-
node.props.clear()
195-
node.clear_properties()
196-
# del node.node.findall(vos.Node.PROPERTIES)[0:]
193+
node.props.clear()
194+
node.clear_properties()
197195
if 'readgroup' in props:
198196
node.chrgrp(props['readgroup'])
199197
if 'writegroup' in props:
200198
node.chwgrp(props['writegroup'])
201199
if 'ispublic' in props:
202200
node.set_public(props['ispublic'])
203201
logging.debug("Node: {0}".format(node))
204-
status = client.update(node, opt.recursive)
205-
if status:
206-
sys.exit(status)
202+
successes, failures = client.update(node, opt.recursive)
203+
if opt.recursive:
204+
if failures:
205+
logging.error('WARN. updated count: {}, failed count: {}\n'.
206+
format(successes, failures))
207+
sys.exit(-1)
208+
else:
209+
logging.info('DONE. updated count: {}\n'.format(successes))
207210
except Exception as ex:
208211
exit_on_exception(ex)
209212

‎vos/vos/commands/vcp.py‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# ****************** CANADIAN ASTRONOMY DATA CENTRE *******************
33
# ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
44
#
5-
# (c) 2022. (c) 2022.
5+
# (c) 2024. (c) 2024.
66
# Government of Canada Gouvernement du Canada
77
# National Research Council Conseil national de recherches
88
# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
@@ -69,7 +69,7 @@
6969
"""copy files vospace to local or local to VOSpace"""
7070
from .. import md5_cache
7171
from .. import vos
72-
from ..commonparser import CommonParser, set_logging_level_from_args,\
72+
from ..commonparser import CommonParser, set_logging_level_from_args, \
7373
exit_on_exception, URI_DESCRIPTION
7474

7575
try:
@@ -324,19 +324,6 @@ def copy(source_name, destination_name, exclude=None, include=None,
324324
if ans != 'y':
325325
raise Exception("File exists")
326326

327-
if not access(os.path.dirname(destination_name), os.F_OK):
328-
raise OSError(errno.EEXIST,
329-
"vcp: ContainerNode %s does not exist" %
330-
os.path.dirname(
331-
destination_name))
332-
333-
if not isdir(os.path.dirname(destination_name)) and not islink(
334-
os.path.dirname(destination_name)):
335-
raise OSError(errno.ENOTDIR,
336-
"vcp: %s is not a ContainerNode or LinkNode"
337-
% os.path.dirname(
338-
destination_name))
339-
340327
skip = False
341328
if exclude is not None:
342329
for thisIgnore in exclude.split(','):

‎vos/vos/commands/vrm.py‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# ****************** CANADIAN ASTRONOMY DATA CENTRE *******************
33
# ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
44
#
5-
# (c) 2022. (c) 2022.
5+
# (c) 2024. (c) 2024.
66
# Government of Canada Gouvernement du Canada
77
# National Research Council Conseil national de recherches
88
# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
@@ -68,6 +68,8 @@
6868

6969
"""remove a vospace data node, fails if container node or node is locked."""
7070
import logging
71+
import sys
72+
7173
from ..commonparser import set_logging_level_from_args, exit_on_exception, \
7274
CommonParser, URI_DESCRIPTION
7375
from .. import vos
@@ -81,8 +83,12 @@
8183

8284
def vrm():
8385
parser = CommonParser(description=DESCRIPTION)
86+
parser.add_argument(
87+
"-R", "--recursive", action="store_true",
88+
help="Delete a file or directory even if it's not empty.",
89+
default=False)
8490
parser.add_argument('node',
85-
help='dataNode or linkNode to delete from VOSpace',
91+
help='file, link or possibly directory to delete from VOSpace',
8692
nargs='+')
8793

8894
args = parser.parse_args()
@@ -97,17 +103,27 @@ def vrm():
97103
if not client.is_remote_file(node):
98104
raise Exception(
99105
'{} is not a valid VOSpace handle'.format(node))
100-
if not node.endswith('/'):
101-
if client.get_node(node).islink():
102-
logging.info('deleting link {}'.format(node))
103-
client.delete(node)
104-
elif client.isfile(node):
105-
logging.info('deleting {}'.format(node))
106-
client.delete(node)
107-
elif client.isdir(node):
108-
raise Exception('{} is a directory'.format(node))
106+
if args.recursive:
107+
successes, failures = client.recursive_delete(node)
108+
if failures:
109+
logging.error('WARN. deleted count: {}, failed count: '
110+
'{}\n'.format(successes, failures))
111+
sys.exit(-1)
112+
else:
113+
logging.info(
114+
'DONE. deleted count: {}\n'.format(successes))
109115
else:
110-
raise Exception('{} is not a directory'.format(node))
116+
if not node.endswith('/'):
117+
if client.get_node(node).islink():
118+
logging.info('deleting link {}'.format(node))
119+
client.delete(node)
120+
elif client.isfile(node):
121+
logging.info('deleting {}'.format(node))
122+
client.delete(node)
123+
elif client.isdir(node):
124+
raise Exception('{} is a directory'.format(node))
125+
else:
126+
raise Exception('{} is not a directory'.format(node))
111127

112128
except Exception as ex:
113129
exit_on_exception(ex)

‎vos/vos/commands/vtag.py‎

Lines changed: 49 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# ****************** CANADIAN ASTRONOMY DATA CENTRE *******************
33
# ************* CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
44
#
5-
# (c) 2022. (c) 2022.
5+
# (c) 2024. (c) 2024.
66
# Government of Canada Gouvernement du Canada
77
# National Research Council Conseil national de recherches
88
# Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
@@ -71,7 +71,9 @@
7171
7272
The tag system is meant to allow tags, in addition to the standard node
7373
properties. """
74+
import logging
7475
import pprint
76+
import sys
7577
from ..commonparser import CommonParser, set_logging_level_from_args, \
7678
exit_on_exception, URI_DESCRIPTION
7779
from .. import vos
@@ -107,14 +109,15 @@ def vtag():
107109
nargs="*")
108110
parser.add_option('--remove', action="store_true",
109111
help='remove the listed property')
112+
parser.add_option('-R', '--recursive', action="store_true",
113+
help='perform the operation recursively on all the descendants')
110114

111-
opt = parser.parse_args()
112-
args = opt
115+
args = parser.parse_args()
113116
set_logging_level_from_args(args)
114117

115118
# the node should be the first argument, the rest should contain
116119
# the key/val pairs
117-
node = args.node
120+
node_arg = args.node
118121

119122
props = []
120123
if args.remove:
@@ -128,35 +131,53 @@ def vtag():
128131

129132
try:
130133
client = vos.Client(
131-
vospace_certfile=opt.certfile,
132-
vospace_token=opt.token,
133-
insecure=opt.insecure)
134-
node = client.get_node(node)
134+
vospace_certfile=args.certfile,
135+
vospace_token=args.token,
136+
insecure=args.insecure)
137+
node = client.get_node(node_arg)
135138
if len(props) == 0:
136139
# print all properties
137140
pprint.pprint(node.props)
138-
139-
changed = False
140-
for prop in props:
141-
prop = prop.split('=')
142-
if len(prop) == 1:
143-
# get one property
144-
pprint.pprint(node.props.get(prop[0], None))
145-
elif len(prop) == 2:
146-
# update properties
147-
key, value = prop
141+
if args.recursive:
142+
node.props.clear()
143+
node.clear_properties()
144+
for prop in props:
145+
key, value = prop.split('=')
148146
if len(value) == 0:
149147
value = None
150-
if value != node.props.get(key, None):
151-
node.props[key] = value
152-
changed = True
153-
else:
154-
raise ValueError(
155-
"Illegal keyword of value character ('=') used: %s" % (
156-
'='.join(prop)))
157-
158-
if changed:
159-
client.add_props(node)
148+
node.props[key] = value
149+
successes, failures = client.add_props(node, recursive=True)
150+
if args.recursive:
151+
if failures:
152+
logging.error(
153+
'WARN. updated count: {}, failed count: {}\n'.
154+
format(successes, failures))
155+
sys.exit(-1)
156+
else:
157+
logging.info(
158+
'DONE. updated count: {}\n'.format(successes))
159+
else:
160+
changed = False
161+
for prop in props:
162+
prop = prop.split('=')
163+
if len(prop) == 1:
164+
# get one property
165+
pprint.pprint(node.props.get(prop[0], None))
166+
elif len(prop) == 2:
167+
# update properties
168+
key, value = prop
169+
if len(value) == 0:
170+
value = None
171+
if value != node.props.get(key, None):
172+
node.props[key] = value
173+
changed = True
174+
else:
175+
raise ValueError(
176+
"Illegal keyword of value character ('=') used: %s" % (
177+
'='.join(prop)))
178+
179+
if changed:
180+
client.add_props(node)
160181
except Exception as ex:
161182
exit_on_exception(ex)
162183

‎vos/vos/tests/test_config.py‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,12 @@
6565
#
6666
# ***********************************************************************
6767
#
68-
68+
import unittest
6969
# Test the vosconfig functionality
7070
from unittest.mock import Mock, patch
7171
import warnings
7272

73-
from vos import vosconfig, Connection
73+
from vos import vosconfig
7474
import pytest
7575
import tempfile
7676
import os
@@ -80,29 +80,30 @@
8080
skipTests = False
8181

8282

83+
# @unittest.skip('Functionality no longer used')
8384
@patch('vos.vos.os.path.exists', Mock())
8485
def test_update_config():
8586
# Cause all warnings to always be triggered.
8687
warnings.simplefilter("always")
87-
with patch('vos.vos.open') as open_mock:
88+
with patch('builtins.open') as open_mock:
8889
old_content = 'blah'
8990
new_config_mock = Mock()
9091
open_mock.return_value.read.return_value = old_content
9192
open_mock.return_value.write = new_config_mock
9293
vosconfig._update_config()
93-
assert new_config_mock.called_once_with(old_content)
94+
new_config_mock.assert_called_once_with(old_content)
9495

9596
# test rewrite vospace resource in config file
9697
new_config_mock.reset_mock()
9798
# Cause all warnings to always be triggered.
9899
warnings.simplefilter("always")
99-
with patch('vos.vos.open') as open_mock:
100+
with patch('builtins.open') as open_mock:
100101
old_content = 'blah\nresourceID=ivo://cadc.nrc.ca/vospace\nfoo'
101102
new_content = Mock()
102103
open_mock.return_value.read.return_value = old_content
103104
open_mock.return_value.write = new_content
104105
vosconfig._update_config()
105-
assert new_config_mock.called_once_with(old_content.replace(
106+
new_content.assert_called_once_with(old_content.replace(
106107
'vospace', 'vault'))
107108

108109
# test rewrite transfer protocol in config file
@@ -111,14 +112,14 @@ def test_update_config():
111112
"# transfer protocol configuration is no longer supported\n"
112113
# Cause all warnings to always be triggered.
113114
warnings.simplefilter("always")
114-
with patch('vos.vos.open') as open_mock:
115+
with patch('builtins.open') as open_mock:
115116
old_content = 'blah\nprotocol=http\nfoo'
116117
new_content = Mock()
117118
open_mock.return_value.read.return_value = old_content
118119
open_mock.return_value.write = new_content
119120
vosconfig._update_config()
120-
assert new_config_mock.called_once_with(old_content.replace(
121-
'protocol', '{}#protocol'.format(protocol_text)))
121+
new_content.assert_called_once_with(old_content.replace(
122+
'protocol', '{}# protocol'.format(protocol_text)))
122123

123124

124125
@patch('vos.vosconfig.os.path.isfile', Mock())

0 commit comments

Comments
 (0)