diff --git a/chef/__init__.py b/chef/__init__.py index 2c0645b..4cee954 100644 --- a/chef/__init__.py +++ b/chef/__init__.py @@ -5,6 +5,7 @@ from chef.api import ChefAPI, autoconfigure from chef.client import Client from chef.data_bag import DataBag, DataBagItem +from chef.encrypted_data_bag_item import EncryptedDataBagItem from chef.exceptions import ChefError from chef.node import Node from chef.role import Role diff --git a/chef/aes.py b/chef/aes.py new file mode 100644 index 0000000..31d2e12 --- /dev/null +++ b/chef/aes.py @@ -0,0 +1,129 @@ +import os +import sys + +from ctypes import * +from chef.rsa import _eay, SSLError + +c_int_p = POINTER(c_int) + +# void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a); +EVP_CIPHER_CTX_init = _eay.EVP_CIPHER_CTX_init +EVP_CIPHER_CTX_init.argtypes = [c_void_p] +EVP_CIPHER_CTX_init.restype = None + +#int EVP_CipherUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, +# int *outl, unsigned char *in, int inl); +EVP_CipherUpdate = _eay.EVP_CipherUpdate +EVP_CipherUpdate.argtypes = [c_void_p, c_char_p, c_int_p, c_char_p, c_int] +EVP_CipherUpdate.restype = c_int + +#int EVP_CipherFinal(EVP_CIPHER_CTX *ctx, unsigned char *out, +# int *outl); +EVP_CipherFinal = _eay.EVP_CipherFinal +EVP_CipherFinal.argtypes = [c_void_p, c_char_p, c_int_p] +EVP_CipherFinal.restype = c_int + +#EVP_CIPHER *EVP_aes_256_cbc(void); +EVP_aes_256_cbc = _eay.EVP_aes_256_cbc +EVP_aes_256_cbc.argtypes = [] +EVP_aes_256_cbc.restype = c_void_p + +#EVP_MD *EVP_sha1(void); +EVP_sha1 = _eay.EVP_sha1 +EVP_sha1.argtypes = [] +EVP_sha1.restype = c_void_p + +#int EVP_CipherInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type, +# unsigned char *key, unsigned char *iv, int enc); +EVP_CipherInit = _eay.EVP_CipherInit +EVP_CipherInit.argtypes = [c_void_p, c_void_p, c_char_p, c_char_p, c_int] +EVP_CipherInit.restype = c_int + +#int EVP_CIPHER_CTX_set_padding(EVP_CIPHER_CTX *x, int padding); +EVP_CIPHER_CTX_set_padding = _eay.EVP_CIPHER_CTX_set_padding +EVP_CIPHER_CTX_set_padding.argtypes = [c_void_p, c_int] +EVP_CIPHER_CTX_set_padding.restype = c_int + +# Structures required for ctypes + +EVP_MAX_IV_LENGTH = 16 +EVP_MAX_BLOCK_LENGTH = 32 +AES_BLOCK_SIZE = 16 + +class EVP_CIPHER(Structure): + _fields_ = [ + ("nid", c_int), + ("block_size", c_int), + ("key_len", c_int), + ("iv_len", c_int), + ("flags", c_ulong), + ("init", c_voidp), + ("do_cipher", c_voidp), + ("cleanup", c_voidp), + ("set_asn1_parameters", c_voidp), + ("get_asn1_parameters", c_voidp), + ("ctrl", c_voidp), + ("app_data", c_voidp) + ] + +class EVP_CIPHER_CTX(Structure): + _fields_ = [ + ("cipher", POINTER(EVP_CIPHER)), + ("engine", c_voidp), + ("encrypt", c_int), + ("buflen", c_int), + ("oiv", c_ubyte * EVP_MAX_IV_LENGTH), + ("iv", c_ubyte * EVP_MAX_IV_LENGTH), + ("buf", c_ubyte * EVP_MAX_BLOCK_LENGTH), + ("num", c_int), + ("app_data", c_voidp), + ("key_len", c_int), + ("flags", c_ulong), + ("cipher_data", c_voidp), + ("final_used", c_int), + ("block_mask", c_int), + ("final", c_ubyte * EVP_MAX_BLOCK_LENGTH) ] + + +class AES256Cipher(object): + def __init__(self, key, iv, salt='12345678'): + self.key_data = create_string_buffer(key) + self.iv = create_string_buffer(iv) + self.encryptor = self.decryptor = None + self.salt = create_string_buffer(salt.encode('utf8')) + + self.encryptor = EVP_CIPHER_CTX() + self._init_cipher(byref(self.encryptor), 1) + + self.decryptor = EVP_CIPHER_CTX() + self._init_cipher(byref(self.decryptor), 0) + + def _init_cipher(self, ctypes_cipher, crypt_mode): + """ crypt_mode parameter is a flag deciding whether the cipher should be + used for encryption (1) or decryption (0) """ + EVP_CIPHER_CTX_init(ctypes_cipher) + EVP_CipherInit(ctypes_cipher, EVP_aes_256_cbc(), self.key_data, self.iv, c_int(crypt_mode)) + EVP_CIPHER_CTX_set_padding(ctypes_cipher, c_int(1)) + + def _process_data(self, ctypes_cipher, data): + # Guard against str passed in when using python3 + if sys.version_info[0] > 2 and isinstance(data, str): + data = data.encode('utf8') + length = c_int(len(data)) + buf_length = c_int(length.value + AES_BLOCK_SIZE) + buf = create_string_buffer(buf_length.value) + + final_buf = create_string_buffer(AES_BLOCK_SIZE) + final_length = c_int(0) + + EVP_CipherUpdate(ctypes_cipher, buf, byref(buf_length), create_string_buffer(data), length) + EVP_CipherFinal(ctypes_cipher, final_buf, byref(final_length)) + + return string_at(buf, buf_length) + string_at(final_buf, final_length) + + + def encrypt(self, data): + return self._process_data(byref(self.encryptor), data) + + def decrypt(self, data): + return self._process_data(byref(self.decryptor), data) diff --git a/chef/api.py b/chef/api.py index 870592f..40ca6cd 100644 --- a/chef/api.py +++ b/chef/api.py @@ -50,13 +50,19 @@ class ChefAPI(object): with ChefAPI('http://localhost:4000', 'client.pem', 'admin'): n = Node('web1') + + In order to use :class:`EncryptedDataBagItem` object it is necessary + to specify either a path to a file containing the Chef secret key and + the Encrypted Databag version to be used (v1 by default). + If both secret_file and secret_key are passed as argument, secret_key + will take precedence. """ ruby_value_re = re.compile(r'#\{([^}]+)\}') env_value_re = re.compile(r'ENV\[(.+)\]') ruby_string_re = re.compile(r'^\s*(["\'])(.*?)\1\s*$') - def __init__(self, url, key, client, version='0.10.8', headers={}, ssl_verify=True): + def __init__(self, url, key, client, version='0.10.8', headers={}, ssl_verify=True, secret_file=None, secret_key=None, encryption_version=1): self.url = url.rstrip('/') self.parsed_url = six.moves.urllib.parse.urlparse(self.url) if not isinstance(key, Key): @@ -67,11 +73,22 @@ def __init__(self, url, key, client, version='0.10.8', headers={}, ssl_verify=Tr self.client = client self.version = version self.headers = dict((k.lower(), v) for k, v in six.iteritems(headers)) + self.encryption_version = encryption_version self.version_parsed = pkg_resources.parse_version(self.version) self.platform = self.parsed_url.hostname == 'api.opscode.com' self.ssl_verify = ssl_verify if not api_stack_value(): self.set_default() + self.encryption_key = None + # Read the secret key from the input file + if secret_file is not None: + self.secret_file = secret_file + if os.path.exists(self.secret_file): + self.encryption_key = open(self.secret_file).read().strip() + if secret_key is not None: + if encryption_key is not None: + log.debug('Two encryption key found (file and parameter). The key passed as parameter will be used') + self.encryption_key = secret_key @classmethod def from_config_file(cls, path): @@ -83,8 +100,8 @@ def from_config_file(cls, path): # Can't even read the config file log.debug('Unable to read config file "%s"', path) return - url = key_path = client_name = None ssl_verify = True + url = key_path = client_name = encryption_version = None for line in open(path): if not line.strip() or line.startswith('#'): continue # Skip blanks and comments @@ -123,6 +140,9 @@ def _ruby_value(match): elif key == 'node_name': log.debug('Found client name: %r', value) client_name = value + elif key == 'data_bag_encrypt_version': + log.debug('Found data bag encryption version: %r', value) + encryption_version = value elif key == 'client_key': log.debug('Found key path: %r', value) key_path = value @@ -161,7 +181,9 @@ def _ruby_value(match): return if not client_name: client_name = socket.getfqdn() - return cls(url, key_path, client_name, ssl_verify=ssl_verify) + if not encryption_version: + encryption_version = 1 + return cls(url, key_path, client_name, ssl_verify=ssl_verify, encryption_version=encryption_version) @staticmethod def get_global(): diff --git a/chef/encrypted_data_bag_item.py b/chef/encrypted_data_bag_item.py new file mode 100644 index 0000000..6d3e65f --- /dev/null +++ b/chef/encrypted_data_bag_item.py @@ -0,0 +1,153 @@ +from chef.exceptions import ChefUnsupportedEncryptionVersionError, ChefDecryptionError +from chef.aes import AES256Cipher, EVP_MAX_IV_LENGTH +from chef.utils import json +from chef.data_bag import DataBagItem + +import os +import sys +import hmac +import base64 +import chef +import hashlib +import binascii +import itertools +import six +from six.moves import filterfalse, zip_longest + + +class EncryptedDataBagItem(DataBagItem): + """An Encrypted Chef data bag item object. + + Encrypted Databag Items behave in the same way as :class:`DatabagItem` + except the keys and values are encrypted as detailed in the Chef docs: + https://docs.chef.io/data_bags.html#encrypt-a-data-bag-item + + Refer to the :class:`DatabagItem` documentation for usage. + """ + SUPPORTED_ENCRYPTION_VERSIONS = (1,2) + AES_MODE = 'aes_256_cbc' + + def __getitem__(self, key): + if key == 'id': + return self.raw_data[key] + else: + return create_decryptor(self.api.encryption_key, self.raw_data[key]).decrypt() + + def __setitem__(self, key, value): + if key == 'id': + self.raw_data[key] = value + else: + self.raw_data[key] = create_encryptor(self.api.encryption_key, value, self.api.encryption_version).to_dict() + +def create_encryptor(key, data, version): + try: + return { + 1: EncryptorVersion1(key, data), + 2: EncryptorVersion2(key, data) + }[version] + except KeyError: + raise ChefUnsupportedEncryptionVersionError(version) + +class EncryptorVersion1(object): + VERSION = 1 + + def __init__(self, key, data): + self.plain_key = key.encode('utf8') + self.key = hashlib.sha256(key.encode('utf8')).digest() + self.data = data + self.iv = binascii.hexlify(os.urandom(int(EVP_MAX_IV_LENGTH/2))) + self.encryptor = AES256Cipher(key=self.key, iv=self.iv) + self.encrypted_data = None + + def encrypt(self): + if self.encrypted_data is None: + data = json.dumps({'json_wrapper': self.data}) + self.encrypted_data = self.encryptor.encrypt(data) + return self.encrypted_data + + def to_dict(self): + return { + "encrypted_data": base64.standard_b64encode(self.encrypt()).decode('utf8'), + "iv": base64.standard_b64encode(self.iv).decode('utf8'), + "version": self.VERSION, + "cipher": "aes-256-cbc" + } + +class EncryptorVersion2(EncryptorVersion1): + VERSION = 2 + + def __init__(self, key, data): + super(EncryptorVersion2, self).__init__(key, data) + self.hmac = None + + def encrypt(self): + self.encrypted_data = super(EncryptorVersion2, self).encrypt() + self.hmac = (self.hmac if self.hmac is not None else self._generate_hmac()) + return self.encrypted_data + + def _generate_hmac(self): + raw_hmac = hmac.new(self.plain_key, base64.standard_b64encode(self.encrypted_data), hashlib.sha256).digest() + return raw_hmac + + def to_dict(self): + result = super(EncryptorVersion2, self).to_dict() + result['hmac'] = base64.standard_b64encode(self.hmac).decode('utf8') + return result + +def get_decryption_version(data): + if 'version' in data: + if str(data['version']) in map(str, EncryptedDataBagItem.SUPPORTED_ENCRYPTION_VERSIONS): + return data['version'] + else: + raise ChefUnsupportedEncryptionVersionError(data['version']) + else: + return 1 + +def create_decryptor(key, data): + version = get_decryption_version(data) + if version == 1: + return DecryptorVersion1(key, data['encrypted_data'], data['iv']) + elif version == 2: + return DecryptorVersion2(key, data['encrypted_data'], data['iv'], data['hmac']) + +class DecryptorVersion1(object): + def __init__(self, key, data, iv): + self.key = hashlib.sha256(key.encode('utf8')).digest() + self.data = base64.standard_b64decode(data) + self.iv = base64.standard_b64decode(iv) + self.decryptor = AES256Cipher(key=self.key, iv=self.iv) + + def decrypt(self): + value = self.decryptor.decrypt(self.data) + # After decryption we should get a string with JSON + try: + value = json.loads(value.decode('utf-8')) + except ValueError: + raise ChefDecryptionError("Error decrypting data bag value. Most likely the provided key is incorrect") + return value['json_wrapper'] + +class DecryptorVersion2(DecryptorVersion1): + def __init__(self, key, data, iv, hmac): + super(DecryptorVersion2, self).__init__(key, data, iv) + self.hmac = base64.standard_b64decode(hmac) + self.encoded_data = data + + def _validate_hmac(self): + encoded_data = self.encoded_data.encode('utf8') + + expected_hmac = hmac.new(self.key, encoded_data, hashlib.sha256).digest() + valid = len(expected_hmac) ^ len(self.hmac) + for expected_char, candidate_char in zip_longest(expected_hmac, self.hmac): + if sys.version_info[0] > 2: + valid |= expected_char ^ candidate_char + else: + valid |= ord(expected_char) ^ ord(candidate_char) + + return valid == 0 + + def decrypt(self): + if self._validate_hmac(): + return super(DecryptorVersion2, self).decrypt() + else: + raise ChefDecryptionError("Error decrypting data bag value. HMAC validation failed.") + diff --git a/chef/exceptions.py b/chef/exceptions.py index b32ed3f..c1ff004 100644 --- a/chef/exceptions.py +++ b/chef/exceptions.py @@ -31,7 +31,14 @@ class ChefServerNotFoundError(ChefServerError): class ChefAPIVersionError(ChefError): """An incompatible API version error""" - class ChefObjectTypeError(ChefError): """An invalid object type error""" +class ChefUnsupportedEncryptionVersionError(ChefError): + def __init__(self, version): + message = "This version of chef does not support encrypted data bag item format version %s" % version + return super(ChefError, self).__init__(message) + +class ChefDecryptionError(ChefError): + """Error decrypting data bag value. Most likely the provided key is incorrect""" + diff --git a/chef/tests/configs/encryption.rb b/chef/tests/configs/encryption.rb new file mode 100644 index 0000000..68528fe --- /dev/null +++ b/chef/tests/configs/encryption.rb @@ -0,0 +1,5 @@ +chef_server_url 'http://chef:4000' +client_key '../client.pem' +node_name "test_1" + +data_bag_encrypt_version '2' diff --git a/chef/tests/encryption_key b/chef/tests/encryption_key new file mode 100644 index 0000000..440928e --- /dev/null +++ b/chef/tests/encryption_key @@ -0,0 +1 @@ +1FNnCAvhritIPDerwUYKFNFZ8NaAIWyCXV43hqSVDeGk5pDt253E+RlUHlL7H/3HEFo/gnZWsk9Y5bEyOP7tUQSnT8enCbFqtvyBpiVep+4BYHss2aWBqqsm7aiPXa+BQHagmVHySleU+sFdLcNOASNMLiUB6azk8Xme1Gris8Awavrn/s5vRB7Bsl7xl84nSmu7Lg3C6Vezyye6K4ZmJOA1p0QPSMVGEJC5RkwAmA+W6G5MilBDMdxxN7mxy49WRSFLT35xFQNJOJ+Rvk53FJrhOCmiHkVNumF2MuhIpLsbrqpcdsU5UIxibjd2Dt+yz7/qytCsGSyZkVws09MgAH5icjZYV6DL8Y9CRa39KEyHl5DjHmWiRiuoFTc6oiUa0QAh08X64jz8OvcTWCJD9Fi5PdNkJblDMp9g6vvn/UPTos2s0KjzkLKdRbLrJovCSs52kkhTzfYXOYt4rmi5mQbdtcr2vsXFs+CT68Yfs56RFA2BA/+KLdaNzHFeH/Wl3h/hrciQfpAW62jnttBGr7sMV0pevXQTr2npPWq0fZHWO4gxkrL729najiDPOEeA2TeHV6+h6znZNYvfpNIRPIOMDLG7bdq2+/G7OvuE7u15qHYzWlJpvouhLA55upDK6CK1ONQw14JIK4+s9Dt2gYpV//G7MqnFMsnq3Y9ptt4= \ No newline at end of file diff --git a/chef/tests/test_aes.py b/chef/tests/test_aes.py new file mode 100644 index 0000000..38c73d6 --- /dev/null +++ b/chef/tests/test_aes.py @@ -0,0 +1,24 @@ +from chef.tests import ChefTestCase, TEST_ROOT +from chef.aes import AES256Cipher +from chef.rsa import SSLError + +import base64 +import os +import hashlib +import json + +class AES256CipherTestCase(ChefTestCase): + def setUp(self): + super(AES256CipherTestCase, self).setUp() + enc_key = open(os.path.join(TEST_ROOT, 'encryption_key')).read() + key = hashlib.sha256(enc_key.encode('utf8')).digest() + iv = base64.standard_b64decode('GLVikZLxG0SWYnb68Pr8Ag==\n') + self.cipher = AES256Cipher(key, iv) + + def test_encrypt(self): + encrypted_value = self.cipher.encrypt('{"json_wrapper":"secr3t c0d3"}') + self.assertEquals(base64.standard_b64encode(encrypted_value).strip(), "Ym5T8umtSd0wgjDYq1ZDK5dAh6OjgrTxlloGNf2xYhg=".encode('utf8')) + + def test_decrypt(self): + decrypted_value = self.cipher.decrypt(base64.standard_b64decode('Ym5T8umtSd0wgjDYq1ZDK5dAh6OjgrTxlloGNf2xYhg=\n')) + self.assertEquals(decrypted_value, '{"json_wrapper":"secr3t c0d3"}'.encode('utf8')) diff --git a/chef/tests/test_api.py b/chef/tests/test_api.py index 6b19047..579d546 100644 --- a/chef/tests/test_api.py +++ b/chef/tests/test_api.py @@ -48,3 +48,7 @@ def test_bad_key_raises(self): for item in invalids: self.assertRaises( ValueError, ChefAPI, 'foobar', item, 'user') + + def test_encryption(self): + api = self.load('encryption.rb') + self.assertEqual(api.encryption_version, '2') diff --git a/chef/tests/test_encrypted_data_bag_item.py b/chef/tests/test_encrypted_data_bag_item.py new file mode 100644 index 0000000..eea93c3 --- /dev/null +++ b/chef/tests/test_encrypted_data_bag_item.py @@ -0,0 +1,86 @@ +from chef import DataBag, EncryptedDataBagItem +from chef.exceptions import ChefError, ChefUnsupportedEncryptionVersionError, ChefDecryptionError +from chef.tests import ChefTestCase, TEST_ROOT +from chef.api import ChefAPI +from chef.encrypted_data_bag_item import get_decryption_version + +import copy +import os + +class EncryptedDataBagItemTestCase(ChefTestCase): + def setUp(self): + super(EncryptedDataBagItemTestCase, self).setUp() + + """ + This is data encoded using knife, it contains two examples of + encryption methods versions: 1 and 2. + """ + self.knife_examples = { + 'id': 'test', + "pychef_test_ver1": { + "encrypted_data": "Ym5T8umtSd0wgjDYq1ZDK5dAh6OjgrTxlloGNf2xYhg=\n", + "iv": "GLVikZLxG0SWYnb68Pr8Ag==\n", + "version": 1, + "cipher": "aes-256-cbc" + }, + "pychef_test_ver2": { + "encrypted_data": "m2UCN7TYqRJhGfeGFCWtdlF8qtz15W8EmCRqQ4TI4nJpGm/Bqe1WgnzekJus\n7aM0\n", + "hmac": "mzhfGpf/7rkkIQOSbK22zUv1X+bTCNI2l3FgMBgVOAY=\n", + "iv": "EKNLqsxNfiFFDZPDnyXRfw==\n", + "version": 2, + "cipher": "aes-256-cbc" + } + } + + def test__getitem__(self): + api = ChefAPI('https://chef_test:3000', os.path.join(TEST_ROOT, 'client.pem'), 'admin', secret_file=os.path.join(TEST_ROOT, 'encryption_key')) + bag = DataBag('test_1') + item = EncryptedDataBagItem(bag, 'test', api, True) + item.raw_data = copy.deepcopy(self.knife_examples) + + self.assertEqual(item['id'], 'test') + self.assertEqual(item['pychef_test_ver1'], 'secr3t c0d3') + self.assertEqual(item['pychef_test_ver2'], '3ncrypt3d d@t@ b@g') + + # Incorrect IV should raise a decryption error + item.raw_data['pychef_test_ver1']['iv'] = 'ZTM1MjY3OTc4ZjAwOTBlNw==' + self.assertRaises(ChefDecryptionError, item.__getitem__, 'pychef_test_ver1') + + # Invalid HMAC should raise a decryption error + item.raw_data['pychef_test_ver2']['hmac'] = 'v0lMrOmi1ZgA/vtfE2NZO2mO62LagIM2KCZSrWiO/8M=' + self.assertRaises(ChefDecryptionError, item.__getitem__, 'pychef_test_ver2') + + def test__set_item__(self): + api = ChefAPI('https://chef_test:3000', os.path.join(TEST_ROOT, 'client.pem'), 'admin', secret_file=os.path.join(TEST_ROOT, 'encryption_key')) + bag = DataBag('test_1') + item = EncryptedDataBagItem(bag, 'test', api, True) + item['id'] = 'test' + api.encryption_version = 1 + item['pychef_test_ver1'] = 'secr3t c0d3' + api.encryption_version = 2 + item['pychef_test_ver2'] = '3ncrypt3d d@t@ b@g' + + self.assertEqual(item['id'], 'test') + + self.assertIsInstance(item.raw_data['pychef_test_ver1'], dict) + self.assertEqual(item.raw_data['pychef_test_ver1']['version'], 1) + self.assertEqual(item.raw_data['pychef_test_ver1']['cipher'], 'aes-256-cbc') + self.assertIsNotNone(item.raw_data['pychef_test_ver1']['iv']) + self.assertIsNotNone(item.raw_data['pychef_test_ver1']['encrypted_data']) + + self.assertIsInstance(item.raw_data['pychef_test_ver2'], dict) + self.assertEqual(item.raw_data['pychef_test_ver2']['version'], 2) + self.assertEqual(item.raw_data['pychef_test_ver2']['cipher'], 'aes-256-cbc') + self.assertIsNotNone(item.raw_data['pychef_test_ver2']['iv']) + self.assertIsNotNone(item.raw_data['pychef_test_ver2']['hmac']) + self.assertIsNotNone(item.raw_data['pychef_test_ver2']['encrypted_data']) + +class EncryptedDataBagItemHelpersTestCase(ChefTestCase): + def test_get_version(self): + self.assertEqual(get_decryption_version({"version": "1"}), '1') + self.assertEqual(get_decryption_version({"version": 1}), 1) + self.assertEqual(get_decryption_version({"version": "2"}), '2') + self.assertEqual(get_decryption_version({"version": 2}), 2) + self.assertRaises(ChefUnsupportedEncryptionVersionError, get_decryption_version, {"version": 0}) + self.assertRaises(ChefUnsupportedEncryptionVersionError, get_decryption_version, {"version": "not a number"}) +