feat: support for syncing units from libraries to courses (openedx#36553)

* feat: library unit sync
* feat: create component link only for component xblocks
* feat: container link model
* feat: update downstream api views
* feat: delete extra components in container on sync (not working)
* fix: duplicate definitions of LibraryXBlockMetadata
* test: add a new integration test suite for syncing
* feat: partially implement container+child syncing
* fix: blockserializer wasn't always serializing all HTML block fields
* feat: handle reorder, addition and deletion of components in sync

Updates children components of unit in course based on upstream unit,
deletes removed component, adds new ones and updates order as per
upstream.

* feat: return unit upstreamInfo and disallow edits to units in courses that are sourced from a library (#773)
* feat: Add upstream_info to unit
* feat: disallow edits to units in courses that are sourced from a library (#774)

---------

Co-authored-by: Jillian Vogel <jill@opencraft.com>
Co-authored-by: Rômulo Penido <romulo.penido@gmail.com>

* docs: capitalization of XBlock

Co-authored-by: David Ormsbee <dave@axim.org>

* refactor: (minor) change python property name to reflect type better

* fix: lots of "Tried to inspect a missing...upstream link" warnings

when viewing a unit in Studio

* docs: mention potential REST API for future refactor

* fix: check if upstream actually exists before making unit read-only

* chore: fix camel-case var

* fix: test failure when mocked XBlock doesn't have UpstreamSyncMixin

---------

Co-authored-by: Braden MacDonald <braden@opencraft.com>
Co-authored-by: Chris Chávez <xnpiochv@gmail.com>
Co-authored-by: Jillian Vogel <jill@opencraft.com>
Co-authored-by: Rômulo Penido <romulo.penido@gmail.com>
Co-authored-by: Braden MacDonald <mail@bradenm.com>
Co-authored-by: David Ormsbee <dave@axim.org>

Author: 6 people

cms/djangoapps/contentstore/admin.py

@@ -13,15 +13,15 @@
 from cms.djangoapps.contentstore.models import (
     BackfillCourseTabsConfig,
     CleanStaleCertificateAvailabilityDatesConfig,
+    ComponentLink,
+    ContainerLink,
     LearningContextLinksStatus,
-    PublishableEntityLink,
-    VideoUploadConfig
+    VideoUploadConfig,
 )
 from cms.djangoapps.contentstore.outlines_regenerate import CourseOutlineRegenerate
 from openedx.core.djangoapps.content.learning_sequences.api import key_supports_outlines
 
-from .tasks import update_outline_from_modulestore_task, update_all_outlines_from_modulestore_task
-
+from .tasks import update_all_outlines_from_modulestore_task, update_outline_from_modulestore_task
 
 log = logging.getLogger(__name__)
 
@@ -88,10 +88,10 @@ class CleanStaleCertificateAvailabilityDatesConfigAdmin(ConfigurationModelAdmin)
     pass
 
 
-@admin.register(PublishableEntityLink)
-class PublishableEntityLinkAdmin(admin.ModelAdmin):
+@admin.register(ComponentLink)
+class ComponentLinkAdmin(admin.ModelAdmin):
     """
-    PublishableEntityLink admin.
+    ComponentLink admin.
     """
     fields = (
         "uuid",
@@ -127,6 +127,45 @@ def has_change_permission(self, request, obj=None):
         return False
 
 
+@admin.register(ContainerLink)
+class ContainerLinkAdmin(admin.ModelAdmin):
+    """
+    ContainerLink admin.
+    """
+    fields = (
+        "uuid",
+        "upstream_container",
+        "upstream_container_key",
+        "upstream_context_key",
+        "downstream_usage_key",
+        "downstream_context_key",
+        "version_synced",
+        "version_declined",
+        "created",
+        "updated",
+    )
+    readonly_fields = fields
+    list_display = [
+        "upstream_container",
+        "upstream_container_key",
+        "downstream_usage_key",
+        "version_synced",
+        "updated",
+    ]
+    search_fields = [
+        "upstream_container_key",
+        "upstream_context_key",
+        "downstream_usage_key",
+        "downstream_context_key",
+    ]
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+
 @admin.register(LearningContextLinksStatus)
 class LearningContextLinksStatusAdmin(admin.ModelAdmin):
     """

cms/djangoapps/contentstore/helpers.py

@@ -31,7 +31,8 @@
 )
 
 from cms.djangoapps.models.settings.course_grading import CourseGradingModel
-from cms.lib.xblock.upstream_sync import UpstreamLink, UpstreamLinkException, fetch_customizable_fields
+from cms.lib.xblock.upstream_sync import UpstreamLink, UpstreamLinkException
+from cms.lib.xblock.upstream_sync_block import fetch_customizable_fields_from_block
 from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
 import openedx.core.djangoapps.content_staging.api as content_staging_api
 import openedx.core.djangoapps.content_tagging.api as content_tagging_api
@@ -416,7 +417,7 @@ def _fetch_and_set_upstream_link(
     user: User
 ):
     """
-    Fetch and set upstream link for the given xblock. This function handles following cases:
+    Fetch and set upstream link for the given xblock which is being pasted. This function handles following cases:
     * the xblock is copied from a v2 library; the library block is set as upstream.
     * the xblock is copied from a course; no upstream is set, only copied_from_block is set.
     * the xblock is copied from a course where the source block was imported from a library; the original libary block
@@ -425,7 +426,7 @@ def _fetch_and_set_upstream_link(
     # Try to link the pasted block (downstream) to the copied block (upstream).
     temp_xblock.upstream = copied_from_block
     try:
-        UpstreamLink.get_for_block(temp_xblock)
+        upstream_link = UpstreamLink.get_for_block(temp_xblock)
     except UpstreamLinkException:
         # Usually this will fail. For example, if the copied block is a modulestore course block, it can't be an
         # upstream. That's fine! Instead, we store a reference to where this block was copied from, in the
@@ -456,7 +457,8 @@ def _fetch_and_set_upstream_link(
         # later wants to restore it, it will restore to the value that the field had when the block was pasted. Of
         # course, if the author later syncs updates from a *future* published upstream version, then that will fetch
         # new values from the published upstream content.
-        fetch_customizable_fields(upstream=temp_xblock, downstream=temp_xblock, user=user)
+        if isinstance(upstream_link.upstream_key, UsageKey):  # only if upstream is a block, not a container
+            fetch_customizable_fields_from_block(downstream=temp_xblock, user=user, upstream=temp_xblock)
 
 
 def _import_xml_node_to_parent(
@@ -790,3 +792,26 @@ def _get_usage_key_from_node(node, parent_id: str) -> UsageKey | None:
         )
 
     return usage_key
+
+
+def concat_static_file_notices(notices: list[StaticFileNotices]) -> StaticFileNotices:
+    """Combines multiple static file notices into a single object
+
+    Args:
+        notices: list of StaticFileNotices
+
+    Returns:
+        Single StaticFileNotices
+    """
+    new_files = []
+    conflicting_files = []
+    error_files = []
+    for notice in notices:
+        new_files.extend(notice.new_files)
+        conflicting_files.extend(notice.conflicting_files)
+        error_files.extend(notice.error_files)
+    return StaticFileNotices(
+        new_files=list(set(new_files)),
+        conflicting_files=list(set(conflicting_files)),
+        error_files=list(set(error_files)),
+    )

cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py

@@ -1,5 +1,5 @@
 """
-Management command to recreate upstream-dowstream links in PublishableEntityLink for course(s).
+Management command to recreate upstream-dowstream links in ComponentLink for course(s).
 
 This command can be run for all the courses or for given list of courses.
 """
@@ -23,7 +23,7 @@
 
 class Command(BaseCommand):
     """
-    Recreate links for course(s) in PublishableEntityLink table.
+    Recreate upstream links for course(s) in ComponentLink and ContainerLink tables.
 
     Examples:
         # Recreate upstream links for two courses.

cms/djangoapps/contentstore/migrations/0010_container_link_models.py

@@ -0,0 +1,58 @@
+# Generated by Django 4.2.20 on 2025-04-22 15:08
+import uuid
+
+import django.db.models.deletion
+import opaque_keys.edx.django.models
+import openedx_learning.lib.fields
+import openedx_learning.lib.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('oel_components', '0003_remove_componentversioncontent_learner_downloadable'),
+        ('contentstore', '0009_learningcontextlinksstatus_publishableentitylink'),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name='PublishableEntityLink',
+            new_name='ComponentLink',
+        ),
+        migrations.AlterModelOptions(
+            name='componentlink',
+            options={'verbose_name': 'Component Link', 'verbose_name_plural': 'Component Links'},
+        ),
+        migrations.AlterField(
+            model_name='componentlink',
+            name='upstream_block',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name='links',
+                to='oel_components.component',
+            ),
+        ),
+        migrations.CreateModel(
+            name='ContainerLink',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')),
+                ('upstream_context_key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, db_index=True, help_text='Upstream context key i.e., learning_package/library key', max_length=500)),
+                ('downstream_usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255, unique=True)),
+                ('downstream_context_key', opaque_keys.edx.django.models.CourseKeyField(db_index=True, max_length=255)),
+                ('version_synced', models.IntegerField()),
+                ('version_declined', models.IntegerField(blank=True, null=True)),
+                ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])),
+                ('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])),
+                ('upstream_container_key', opaque_keys.edx.django.models.ContainerKeyField(help_text='Upstream block key (e.g. lct:...), this value cannot be null and is useful to track upstream library blocks that do not exist yet or were deleted.', max_length=255)),
+                ('upstream_container', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='links', to='oel_publishing.container')),
+            ],
+            options={
+                'abstract': False,
+                'verbose_name': 'Container Link',
+                'verbose_name_plural': 'Container Links',
+            },
+        ),
+    ]