add support for creating and deleting indexes #125
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,32 @@ | ||
| from django.db.backends.base.introspection import BaseDatabaseIntrospection | ||
| from django.db.models import Index | ||
| from pymongo import ASCENDING, DESCENDING | ||
|
|
||
|
|
||
| class DatabaseIntrospection(BaseDatabaseIntrospection): | ||
| ORDER_DIR = {ASCENDING: "ASC", DESCENDING: "DESC"} | ||
|
|
||
| def table_names(self, cursor=None, include_views=False): | ||
| return sorted([x["name"] for x in self.connection.database.list_collections()]) | ||
|
|
||
| def get_constraints(self, cursor, table_name): | ||
| indexes = self.connection.get_collection(table_name).index_information() | ||
| constraints = {} | ||
| for name, details in indexes.items(): | ||
| # Remove underscore prefix from "_id" columns in primary key index. | ||
| if is_primary_key := name == "_id_": | ||
| name = "id" | ||
| details["key"] = [("id", 1)] | ||
| constraints[name] = { | ||
| "check": False, | ||
| "columns": [field for field, order in details["key"]], | ||
| "definition": None, | ||
| "foreign_key": None, | ||
| "index": True, | ||
| "orders": [self.ORDER_DIR[order] for field, order in details["key"]], | ||
| "primary_key": is_primary_key, | ||
| "type": Index.suffix, | ||
| "unique": details.get("unique", False), | ||
| "options": {}, | ||
| } | ||
| return constraints |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,7 @@ | ||
| from django.db.backends.base.schema import BaseDatabaseSchemaEditor | ||
| from django.db.models import Index | ||
| from pymongo import ASCENDING, DESCENDING | ||
| from pymongo.operations import IndexModel | ||
|
|
||
| from .query import wrap_database_errors | ||
| from .utils import OperationCollector | ||
|
|
@@ -18,11 +21,30 @@ def get_database(self): | |
| @wrap_database_errors | ||
| def create_model(self, model): | ||
| self.get_database().create_collection(model._meta.db_table) | ||
| self._create_model_indexes(model) | ||
| # Make implicit M2M tables. | ||
| for field in model._meta.local_many_to_many: | ||
| if field.remote_field.through._meta.auto_created: | ||
| self.create_model(field.remote_field.through) | ||
|
|
||
| def _create_model_indexes(self, model): | ||
| """ | ||
| Create all indexes (field indexes, index_together, Meta.indexes) for | ||
| the specified model. | ||
| """ | ||
| if not model._meta.managed or model._meta.proxy or model._meta.swapped: | ||
| return | ||
| # Field indexes | ||
| for field in model._meta.local_fields: | ||
| if self._field_should_be_indexed(model, field): | ||
| self._add_field_index(model, field) | ||
| # Meta.index_together (RemovedInDjango51Warning) | ||
| for field_names in model._meta.index_together: | ||
| self._add_composed_index(model, field_names) | ||
| # Meta.indexes | ||
| for index in model._meta.indexes: | ||
| self.add_index(model, index) | ||
|
|
||
| def delete_model(self, model): | ||
| # Delete implicit M2m tables. | ||
| for field in model._meta.local_many_to_many: | ||
|
|
@@ -40,6 +62,9 @@ def add_field(self, model, field): | |
| self.get_collection(model._meta.db_table).update_many( | ||
| {}, [{"$set": {column: self.effective_default(field)}}] | ||
| ) | ||
| # Add an index, if required. | ||
| if self._field_should_be_indexed(model, field): | ||
| self._add_field_index(model, field) | ||
|
|
||
| def _alter_field( | ||
| self, | ||
|
|
@@ -53,15 +78,27 @@ def _alter_field( | |
| strict=False, | ||
| ): | ||
| collection = self.get_collection(model._meta.db_table) | ||
| # Removed an index? | ||
| old_field_indexed = self._field_should_be_indexed(model, old_field) | ||
| new_field_indexed = self._field_should_be_indexed(model, new_field) | ||
| if old_field_indexed and not new_field_indexed: | ||
| self._remove_field_index(model, old_field) | ||
| # Have they renamed the column? | ||
| if old_field.column != new_field.column: | ||
| collection.update_many({}, {"$rename": {old_field.column: new_field.column}}) | ||
| # Move index to the new field, if needed. | ||
| if old_field_indexed and new_field_indexed: | ||
| self._remove_field_index(model, old_field) | ||
| self._add_field_index(model, new_field) | ||
|
Comment on lines
+91
to
+92
There was a problem hiding this comment. NIT: Add then remove. A dropped index has no further use, but adding one may need to be used quickly. There was a problem hiding this comment. Is the scenario you imagine zero-downtime migrations? In that case, I think the developer would be advised to roll out schema changes in advance of the application logic that uses those fields, and thus we shouldn't worry about this sort of optimization. Otherwise, a site should be in maintenance mode while deploying a new version that requires schema changes. There was a problem hiding this comment. Ahhh, okay. That was the exact case I had been thinking of. Thought of it as a small optimization but if that's not standard practice I'm fine leaving it as is. |
||
| # Replace NULL with the field default if the field and was changed from | ||
| # NULL to NOT NULL. | ||
| if new_field.has_default() and old_field.null and not new_field.null: | ||
| column = new_field.column | ||
| default = self.effective_default(new_field) | ||
| collection.update_many({column: {"$eq": None}}, [{"$set": {column: default}}]) | ||
| # Added an index? | ||
| if not old_field_indexed and new_field_indexed: | ||
| self._add_field_index(model, new_field) | ||
|
|
||
| def remove_field(self, model, field): | ||
| # Remove implicit M2M tables. | ||
|
|
@@ -71,21 +108,99 @@ def remove_field(self, model, field): | |
| # Unset field on existing documents. | ||
| if column := field.column: | ||
| self.get_collection(model._meta.db_table).update_many({}, {"$unset": {column: ""}}) | ||
| if self._field_should_be_indexed(model, field): | ||
| self._remove_field_index(model, field) | ||
|
|
||
| def alter_index_together(self, model, old_index_together, new_index_together): | ||
| olds = {tuple(fields) for fields in old_index_together} | ||
| news = {tuple(fields) for fields in new_index_together} | ||
| # Deleted indexes | ||
| for field_names in olds.difference(news): | ||
| self._remove_composed_index(model, field_names, {"index": True, "unique": False}) | ||
| # Created indexes | ||
| for field_names in news.difference(olds): | ||
| self._add_composed_index(model, field_names) | ||
|
|
||
| def alter_unique_together(self, model, old_unique_together, new_unique_together): | ||
| pass | ||
|
|
||
| def add_index(self, model, index, field=None): | ||
| if index.contains_expressions: | ||
| return | ||
|
Comment on lines
+128
to
+129
There was a problem hiding this comment. Should we not raise a There was a problem hiding this comment. They are ignored (in case the index is useful on another database backend) as per https://docs.djangoproject.com/en/dev/ref/models/indexes/#django.db.models.Index.expressions. |
||
| index_orders = ( | ||
| [(field.column, ASCENDING)] | ||
| if field | ||
| else [ | ||
| # order is "" if ASCENDING or "DESC" if DESCENDING (see | ||
| # django.db.models.indexes.Index.fields_orders). | ||
| (model._meta.get_field(field_name).column, ASCENDING if order == "" else DESCENDING) | ||
| for field_name, order in index.fields_orders | ||
| ] | ||
| ) | ||
| idx = IndexModel(index_orders, name=index.name) | ||
| self.get_collection(model._meta.db_table).create_indexes([idx]) | ||
|
|
||
| def _add_composed_index(self, model, field_names): | ||
| """Add an index on the given list of field_names.""" | ||
| idx = Index(fields=field_names) | ||
| idx.set_name_with_model(model) | ||
| self.add_index(model, idx) | ||
|
|
||
| def _add_field_index(self, model, field): | ||
| """Add an index on a field with db_index=True.""" | ||
| index = Index(fields=[field.name]) | ||
| index.name = self._create_index_name(model._meta.db_table, [field.column]) | ||
| self.add_index(model, index, field=field) | ||
|
|
||
| def remove_index(self, model, index): | ||
| if index.contains_expressions: | ||
| return | ||
| self.get_collection(model._meta.db_table).drop_index(index.name) | ||
|
|
||
| def _remove_composed_index(self, model, field_names, constraint_kwargs): | ||
| """ | ||
| Remove the index on the given list of field_names created by | ||
| index/unique_together, depending on constraint_kwargs. | ||
| """ | ||
| meta_constraint_names = {constraint.name for constraint in model._meta.constraints} | ||
| meta_index_names = {constraint.name for constraint in model._meta.indexes} | ||
| columns = [model._meta.get_field(field).column for field in field_names] | ||
| constraint_names = self._constraint_names( | ||
| model, | ||
| columns, | ||
| exclude=meta_constraint_names | meta_index_names, | ||
| **constraint_kwargs, | ||
| ) | ||
| if len(constraint_names) != 1: | ||
| num_found = len(constraint_names) | ||
| columns_str = ", ".join(columns) | ||
| raise ValueError( | ||
| f"Found wrong number ({num_found}) of constraints for " | ||
| f"{model._meta.db_table}({columns_str})." | ||
| ) | ||
| collection = self.get_collection(model._meta.db_table) | ||
| collection.drop_index(constraint_names[0]) | ||
|
|
||
| def _remove_field_index(self, model, field): | ||
| """Remove a field's db_index=True index.""" | ||
| collection = self.get_collection(model._meta.db_table) | ||
| meta_index_names = {index.name for index in model._meta.indexes} | ||
| index_names = self._constraint_names( | ||
| model, | ||
| [field.column], | ||
| index=True, | ||
| # Retrieve only BTREE indexes since this is what's created with | ||
| # db_index=True. | ||
| type_=Index.suffix, | ||
| exclude=meta_index_names, | ||
| ) | ||
| if len(index_names) != 1: | ||
| num_found = len(index_names) | ||
| raise ValueError( | ||
| f"Found wrong number ({num_found}) of constraints for " | ||
| f"{model._meta.db_table}.{field.column}." | ||
| ) | ||
| collection.drop_index(index_names[0]) | ||
|
|
||
| def add_constraint(self, model, constraint): | ||
| pass | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For users of Django 5.1, the below for loop should include the index_together elements, right?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Support for
Meta.index_togetheris removed in Django 5.1, so these lines will be removed.Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ahhh, looked over the documentation and see that this was deprecated since
4.2