diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index 41d3553..3d1428b 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -1,6 +1,14 @@ name: Lint & Test -on: [push] +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true permissions: contents: read @@ -10,6 +18,12 @@ jobs: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -27,21 +41,22 @@ jobs: black --check . test: + runs-on: ubuntu-latest strategy: matrix: - python_version: 3.8, 3.9, '3.10'] - django_version: [3.2, 4.0, 4.1.0, 4.2.2] + python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + django_version: ["3.2", "4.0", "4.1", "4.2", "5.0"] exclude: - # Ignore Django 4 on Python 3.7 - - python_version: 3.8 - django_version: 4.2.2 - runs-on: ubuntu-latest + - python_version: "3.8" + django_version: "5.0" + - python_version: "3.9" + django_version: "5.0" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python_version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} @@ -54,10 +69,10 @@ jobs: - name: Run tests run: | - coverage3 run --source='./encrypted_fields' manage.py test + coverage run --source="./encrypted_fields" manage.py test coverage xml - name: "Upload coverage to Codecov" - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: fail_ci_if_error: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index abb7406..0000000 --- a/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: python -python: - - "3.9" -env: - - DJANGO_VERSION=3.2 - - DJANGO_VERSION=4.0 - - DJANGO_VERSION=4.1 - - DJANGO_VERSION=4.2 -install: - - pip install -q Django==$DJANGO_VERSION - - pip install -q -r requirements.txt -script: python manage.py test \ No newline at end of file diff --git a/README.md b/README.md index 24c0dc0..e41e9ba 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://api.travis-ci.com/jazzband/django-fernet-encrypted-fields.png)](https://travis-ci.com/jazzband/django-fernet-encrypted-fields) +[![Build Status](https://github.com/jazzband/django-fernet-encrypted-fields/actions/workflows/lint-and-test.yml/badge.svg)](https://github.com/jazzband/django-fernet-encrypted-fields/actions/workflows/lint-and-test.yml) [![Pypi Package](https://badge.fury.io/py/django-fernet-encrypted-fields.png)](http://badge.fury.io/py/django-fernet-encrypted-fields) [![Jazzband](https://jazzband.co/static/img/badge.svg)](https://jazzband.co/) @@ -64,3 +64,4 @@ Currently build in and unit-tested fields. They have the same APIs as their non- | `4.0` |:heavy_check_mark:| | `4.1` |:heavy_check_mark:| | `4.2` |:heavy_check_mark:| +| `5.0` |:heavy_check_mark:| diff --git a/encrypted_fields/fields.py b/encrypted_fields/fields.py index 03e5c45..383d734 100644 --- a/encrypted_fields/fields.py +++ b/encrypted_fields/fields.py @@ -6,12 +6,12 @@ from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC from django.conf import settings -from django.core import validators from django.db import models -from django.db.backends.base.operations import BaseDatabaseOperations from django.utils.functional import cached_property +from django.utils.encoding import force_bytes, force_str -class EncryptedFieldMixin(object): + +class EncryptedFieldMixin: @cached_property def keys(self): keys = [] @@ -21,7 +21,7 @@ def keys(self): else [settings.SALT_KEY] ) for salt_key in salt_keys: - salt = bytes(salt_key, "utf-8") + salt = force_bytes(salt_key) kdf = PBKDF2HMAC( algorithm=hashes.SHA256(), length=32, @@ -30,9 +30,7 @@ def keys(self): backend=default_backend(), ) keys.append( - base64.urlsafe_b64encode( - kdf.derive(settings.SECRET_KEY.encode("utf-8")) - ) + base64.urlsafe_b64encode(kdf.derive(force_bytes(settings.SECRET_KEY))) ) return keys @@ -46,14 +44,14 @@ def get_internal_type(self): """ To treat everything as text """ - return "TextField" + return getattr(self, "_internal_type", "TextField") def get_prep_value(self, value): value = super().get_prep_value(value) if value: if not isinstance(value, str): value = str(value) - return self.f.encrypt(bytes(value, "utf-8")).decode("utf-8") + return force_str(self.f.encrypt(force_bytes(value))) return None def get_db_prep_value(self, value, connection, prepared=False): @@ -64,6 +62,13 @@ def get_db_prep_value(self, value, connection, prepared=False): def from_db_value(self, value, expression, connection): return self.to_python(value) + def decrypt(self, value): + try: + value = force_str(self.f.decrypt(force_bytes(value))) + except (InvalidToken, UnicodeEncodeError): + pass + return value + def to_python(self, value): if ( value is None @@ -71,13 +76,9 @@ def to_python(self, value): or hasattr(self, "_already_decrypted") ): return value - try: - value = self.f.decrypt(bytes(value, "utf-8")).decode("utf-8") - except InvalidToken: - pass - except UnicodeEncodeError: - pass - return super(EncryptedFieldMixin, self).to_python(value) + + value = self.decrypt(value) + return super().to_python(value) def clean(self, value, model_instance): """ @@ -89,6 +90,17 @@ def clean(self, value, model_instance): del self._already_decrypted return ret + @cached_property + def validators(self): + # Temporarily pretend to be whatever type of field we're masquerading + # as, for purposes of constructing validators (needed for + # IntegerField and subclasses). + self._internal_type = super().get_internal_type() + try: + return super().validators + finally: + del self._internal_type + class EncryptedCharField(EncryptedFieldMixin, models.CharField): pass @@ -103,40 +115,7 @@ class EncryptedDateTimeField(EncryptedFieldMixin, models.DateTimeField): class EncryptedIntegerField(EncryptedFieldMixin, models.IntegerField): - @cached_property - def validators(self): - # These validators can't be added at field initialization time since - # they're based on values retrieved from `connection`. - validators_ = [*self.default_validators, *self._validators] - internal_type = models.IntegerField().get_internal_type() - min_value, max_value = BaseDatabaseOperations.integer_field_ranges[internal_type] - if min_value is not None and not any( - ( - isinstance(validator, validators.MinValueValidator) - and ( - validator.limit_value() - if callable(validator.limit_value) - else validator.limit_value - ) - >= min_value - ) - for validator in validators_ - ): - validators_.append(validators.MinValueValidator(min_value)) - if max_value is not None and not any( - ( - isinstance(validator, validators.MaxValueValidator) - and ( - validator.limit_value() - if callable(validator.limit_value) - else validator.limit_value - ) - <= max_value - ) - for validator in validators_ - ): - validators_.append(validators.MaxValueValidator(max_value)) - return validators_ + pass class EncryptedDateField(EncryptedFieldMixin, models.DateField): @@ -160,10 +139,10 @@ def _encrypt_values(self, value): if isinstance(value, dict): return {key: self._encrypt_values(data) for key, data in value.items()} elif isinstance(value, list): - return [self._encrypt_values(data) for data in value] + return [self._encrypt_values(data) for data in value] else: value = str(value) - return self.f.encrypt(bytes(value, "utf-8")).decode("utf-8") + return force_str(self.f.encrypt(force_bytes(value))) def _decrypt_values(self, value): if value is None: @@ -171,10 +150,10 @@ def _decrypt_values(self, value): if isinstance(value, dict): return {key: self._decrypt_values(data) for key, data in value.items()} elif isinstance(value, list): - return [self._decrypt_values(data) for data in value] + return [self._decrypt_values(data) for data in value] else: value = str(value) - return self.f.decrypt(bytes(value, "utf-8")).decode("utf-8") + return force_str(self.f.decrypt(force_bytes(value))) def get_prep_value(self, value): return json.dumps(self._encrypt_values(value=value), cls=self.encoder) @@ -182,17 +161,9 @@ def get_prep_value(self, value): def get_internal_type(self): return "JSONField" - def to_python(self, value): - if ( - value is None - or not isinstance(value, str) - or hasattr(self, "_already_decrypted") - ): - return value + def decrypt(self, value): try: value = self._decrypt_values(value=json.loads(value)) - except InvalidToken: - pass - except UnicodeEncodeError: + except (InvalidToken, UnicodeEncodeError): pass - return super(EncryptedFieldMixin, self).to_python(value) + return value diff --git a/lint-and-test.yml b/lint-and-test.yml deleted file mode 100644 index e3123ce..0000000 --- a/lint-and-test.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Lint & Test - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -permissions: - contents: read - -jobs: - build: - strategy: - matrix: - django_version: [2.2, 3.0, 3.1, 3.2, 4.0a1] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -q Django==${{ matrix.django_version }} - pip install flake8 coverage black - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Lint with Black - run: | - black --check . - - - name: Run tests - run: | - coverage3 run --source='./encrypted_fields' manage.py test - coverage xml - - - name: "Upload coverage to Codecov" - uses: codecov/codecov-action@v2 - with: - fail_ci_if_error: true diff --git a/manage.py b/manage.py index dbe5fea..4a8dc4c 100644 --- a/manage.py +++ b/manage.py @@ -2,7 +2,7 @@ import sys if __name__ == "__main__": - os.environ["DJANGO_SETTINGS_MODULE"] = "package_test.settings" + os.environ["DJANGO_SETTINGS_MODULE"] = "tests.settings" from django.core.management import execute_from_command_line diff --git a/requirements.txt b/requirements.txt index 7cf0930..a2a578b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,2 @@ -cffi==1.14.6 -cryptography==35.0.0 -pycparser==2.20 -Django>=2.2 +cryptography>=35.0.0 +Django>=3.2 \ No newline at end of file diff --git a/setup.py b/setup.py index f1f2ee0..e0ea514 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -from __future__ import print_function from setuptools import setup setup( @@ -6,7 +5,7 @@ description=("This is inspired by django-encrypted-fields."), long_description=open("README.md").read(), long_description_content_type="text/markdown", - url="http://github.com/frgmt/django-fernet-encrypted-fields/", + url="https://github.com/jazzband/django-fernet-encrypted-fields", license="MIT", author="fragment.co.jp", author_email="info@fragment.co.jp", diff --git a/package_test/__init__.py b/tests/__init__.py similarity index 100% rename from package_test/__init__.py rename to tests/__init__.py diff --git a/package_test/models.py b/tests/models.py similarity index 100% rename from package_test/models.py rename to tests/models.py diff --git a/package_test/settings.py b/tests/settings.py similarity index 63% rename from package_test/settings.py rename to tests/settings.py index e169bcb..72b4ee6 100644 --- a/package_test/settings.py +++ b/tests/settings.py @@ -1,6 +1,6 @@ DATABASES = { "default": { - "ENGINE": "django.db.backends.sqlite3", + "ENGINE": "tests.sqlite3", "NAME": ":memory:", }, } @@ -8,7 +8,7 @@ SECRET_KEY = "abc" SALT_KEY = "xyz" -INSTALLED_APPS = ("encrypted_fields", "package_test") +INSTALLED_APPS = ("encrypted_fields", "tests") MIDDLEWARE_CLASSES = [] DEFAULT_AUTO_FIELD = "django.db.models.AutoField" diff --git a/tests/sqlite3/__init__.py b/tests/sqlite3/__init__.py new file mode 100644 index 0000000..e9001e5 --- /dev/null +++ b/tests/sqlite3/__init__.py @@ -0,0 +1 @@ +"""Override to default django SQLite backend required for integer validation test.""" diff --git a/tests/sqlite3/base.py b/tests/sqlite3/base.py new file mode 100644 index 0000000..73ad84e --- /dev/null +++ b/tests/sqlite3/base.py @@ -0,0 +1,16 @@ +from django.db.backends.sqlite3.base import DatabaseWrapper as BaseDatabaseWrapper +from django.db.backends.sqlite3.operations import ( + DatabaseOperations as BaseDatabaseOperations, +) + + +class DatabaseOperations(BaseDatabaseOperations): + def integer_field_range(self, internal_type): + # by default django does not enforce size on SQLite integers + # because it does not + # this is required to pass tests without using a real DB + return self.integer_field_ranges[internal_type] + + +class DatabaseWrapper(BaseDatabaseWrapper): + ops_class = DatabaseOperations diff --git a/package_test/tests.py b/tests/tests.py similarity index 74% rename from package_test/tests.py rename to tests/tests.py index 9d2c9b1..f0a9dce 100644 --- a/package_test/tests.py +++ b/tests/tests.py @@ -1,7 +1,7 @@ import json -import re -from django.db import connection +from django.db.models import CharField +from django.db.models.functions import Cast from django.test import TestCase, override_settings from django.utils import timezone from django.core.exceptions import ValidationError @@ -10,29 +10,28 @@ class FieldTest(TestCase): - def get_db_value(self, field, model_id): - cursor = connection.cursor() - cursor.execute( - "select {0} " - "from package_test_testmodel " - "where id = {1};".format(field, model_id) + def get_db_value(self, field, pk): + queryset = ( + TestModel.objects.filter(pk=pk) + .annotate(raw_field=Cast(field, CharField())) + .values_list("raw_field", flat=True) ) - return cursor.fetchone()[0] + + return queryset.first() def test_char_field_encrypted(self): plaintext = "Oh hi, test reader!" model = TestModel() model.char = plaintext - model.full_clean() model.save() - ciphertext = self.get_db_value("char", model.id) + ciphertext = self.get_db_value("char", model.pk) self.assertNotEqual(plaintext, ciphertext) self.assertTrue("test" not in ciphertext) - fresh_model = TestModel.objects.get(id=model.id) + fresh_model = TestModel.objects.get(id=model.pk) self.assertEqual(fresh_model.char, plaintext) def test_text_field_encrypted(self): @@ -40,15 +39,14 @@ def test_text_field_encrypted(self): model = TestModel() model.text = plaintext - model.full_clean() model.save() - ciphertext = self.get_db_value("text", model.id) + ciphertext = self.get_db_value("text", model.pk) self.assertNotEqual(plaintext, ciphertext) self.assertTrue("test" not in ciphertext) - fresh_model = TestModel.objects.get(id=model.id) + fresh_model = TestModel.objects.get(id=model.pk) self.assertEqual(fresh_model.text, plaintext) def test_datetime_field_encrypted(self): @@ -56,22 +54,20 @@ def test_datetime_field_encrypted(self): model = TestModel() model.datetime = plaintext - model.full_clean() model.save() - ciphertext = self.get_db_value("datetime", model.id) + ciphertext = self.get_db_value("datetime", model.pk) # Django's normal date serialization format - self.assertTrue(re.search("^\d\d\d\d-\d\d-\d\d", ciphertext) is None) + self.assertNotRegex(ciphertext, r"^\d\d\d\d-\d\d-\d\d") - fresh_model = TestModel.objects.get(id=model.id) + fresh_model = TestModel.objects.get(id=model.pk) self.assertEqual(fresh_model.datetime, plaintext) plaintext = "text" with self.assertRaises(ValidationError): model.datetime = plaintext - model.full_clean() model.save() def test_integer_field_encrypted(self): @@ -79,15 +75,14 @@ def test_integer_field_encrypted(self): model = TestModel() model.integer = plaintext - model.full_clean() model.save() - ciphertext = self.get_db_value("integer", model.id) + ciphertext = self.get_db_value("integer", model.pk) self.assertNotEqual(plaintext, ciphertext) self.assertNotEqual(plaintext, str(ciphertext)) - fresh_model = TestModel.objects.get(id=model.id) + fresh_model = TestModel.objects.get(id=model.pk) self.assertEqual(fresh_model.integer, plaintext) # "IntegerField": (-2147483648, 2147483647) @@ -96,7 +91,6 @@ def test_integer_field_encrypted(self): with self.assertRaises(ValidationError): model.integer = plaintext model.full_clean() - model.save() plaintext = "text" @@ -110,11 +104,10 @@ def test_date_field_encrypted(self): model = TestModel() model.date = plaintext - model.full_clean() model.save() - ciphertext = self.get_db_value("date", model.id) - fresh_model = TestModel.objects.get(id=model.id) + ciphertext = self.get_db_value("date", model.pk) + fresh_model = TestModel.objects.get(id=model.pk) self.assertNotEqual(ciphertext, plaintext.isoformat()) self.assertEqual(fresh_model.date, plaintext) @@ -134,19 +127,18 @@ def test_float_field_encrypted(self): model.full_clean() model.save() - ciphertext = self.get_db_value("floating", model.id) + ciphertext = self.get_db_value("floating", model.pk) self.assertNotEqual(plaintext, ciphertext) self.assertNotEqual(plaintext, str(ciphertext)) - fresh_model = TestModel.objects.get(id=model.id) + fresh_model = TestModel.objects.get(id=model.pk) self.assertEqual(fresh_model.floating, plaintext) plaintext = "text" with self.assertRaises(ValueError): model.floating = plaintext - model.full_clean() model.save() def test_email_field_encrypted(self): @@ -154,15 +146,14 @@ def test_email_field_encrypted(self): model = TestModel() model.email = plaintext - model.full_clean() model.save() - ciphertext = self.get_db_value("email", model.id) + ciphertext = self.get_db_value("email", model.pk) self.assertNotEqual(plaintext, ciphertext) self.assertTrue("aron" not in ciphertext) - fresh_model = TestModel.objects.get(id=model.id) + fresh_model = TestModel.objects.get(id=model.pk) self.assertEqual(fresh_model.email, plaintext) plaintext = "text" @@ -170,7 +161,6 @@ def test_email_field_encrypted(self): with self.assertRaises(ValidationError): model.email = plaintext model.full_clean() - model.save() def test_boolean_field_encrypted(self): plaintext = True @@ -180,7 +170,7 @@ def test_boolean_field_encrypted(self): model.full_clean() model.save() - ciphertext = self.get_db_value("boolean", model.id) + ciphertext = self.get_db_value("boolean", model.pk) self.assertNotEqual(plaintext, ciphertext) self.assertNotEqual(True, ciphertext) @@ -190,41 +180,41 @@ def test_boolean_field_encrypted(self): self.assertNotEqual(1, ciphertext) self.assertTrue(not isinstance(ciphertext, bool)) - fresh_model = TestModel.objects.get(id=model.id) + fresh_model = TestModel.objects.get(id=model.pk) self.assertEqual(fresh_model.boolean, plaintext) plaintext = "text" with self.assertRaises(ValidationError): model.boolean = plaintext - model.full_clean() model.save() def test_json_field_encrypted(self): - dict_values = {"key": "value", "list": ["nested", {"key": "val"}], "nested": {"child": "sibling"}} + dict_values = { + "key": "value", + "list": ["nested", {"key": "val"}], + "nested": {"child": "sibling"}, + } model = TestModel() model.json = dict_values - model.full_clean() model.save() - ciphertext = json.loads(self.get_db_value("json", model.id)) - + ciphertext = json.loads(self.get_db_value("json", model.pk)) self.assertNotEqual(dict_values, ciphertext) - fresh_model = TestModel.objects.get(id=model.id) + fresh_model = TestModel.objects.get(id=model.pk) self.assertEqual(fresh_model.json, dict_values) def test_json_field_retains_keys(self): plain_value = {"key": "value", "another_key": "some value"} - + model = TestModel() model.json = plain_value - model.full_clean() model.save() - ciphertext = json.loads(self.get_db_value("json", model.id)) + ciphertext = json.loads(self.get_db_value("json", model.pk)) self.assertEqual(plain_value.keys(), ciphertext.keys()) @@ -244,15 +234,15 @@ def test_rotated_salt(self): model.text = plaintext model.save() - ciphertext = FieldTest.get_db_value(self, "text", model.id) + ciphertext = FieldTest.get_db_value(self, "text", model.pk) self.assertNotEqual(plaintext, ciphertext) self.assertTrue("test" not in ciphertext) - fresh_model = TestModel.objects.get(id=model.id) + fresh_model = TestModel.objects.get(id=model.pk) self.assertEqual(fresh_model.text, plaintext) - old_record = TestModel.objects.get(id=self.original.id) + old_record = TestModel.objects.get(id=self.original.pk) self.assertEqual(fresh_model.text, old_record.text) self.assertNotEqual(