Merge branch 'master' into rpenido/fal-4033-ignore-changes-on-user-state

Author: rpenido

cms/djangoapps/contentstore/admin.py

@@ -13,6 +13,8 @@
 from cms.djangoapps.contentstore.models import (
     BackfillCourseTabsConfig,
     CleanStaleCertificateAvailabilityDatesConfig,
+    LearningContextLinksStatus,
+    PublishableEntityLink,
     VideoUploadConfig
 )
 from cms.djangoapps.contentstore.outlines_regenerate import CourseOutlineRegenerate
@@ -86,6 +88,71 @@ class CleanStaleCertificateAvailabilityDatesConfigAdmin(ConfigurationModelAdmin)
     pass
 
 
+@admin.register(PublishableEntityLink)
+class PublishableEntityLinkAdmin(admin.ModelAdmin):
+    """
+    PublishableEntityLink admin.
+    """
+    fields = (
+        "uuid",
+        "upstream_block",
+        "upstream_usage_key",
+        "upstream_context_key",
+        "downstream_usage_key",
+        "downstream_context_key",
+        "version_synced",
+        "version_declined",
+        "created",
+        "updated",
+    )
+    readonly_fields = fields
+    list_display = [
+        "upstream_block",
+        "upstream_usage_key",
+        "downstream_usage_key",
+        "version_synced",
+        "updated",
+    ]
+    search_fields = [
+        "upstream_usage_key",
+        "upstream_context_key",
+        "downstream_usage_key",
+        "downstream_context_key",
+    ]
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+
+@admin.register(LearningContextLinksStatus)
+class LearningContextLinksStatusAdmin(admin.ModelAdmin):
+    """
+    LearningContextLinksStatus admin.
+    """
+    fields = (
+        "context_key",
+        "status",
+        "created",
+        "updated",
+    )
+    readonly_fields = ("created", "updated")
+    list_display = (
+        "context_key",
+        "status",
+        "created",
+        "updated",
+    )
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+
 admin.site.register(BackfillCourseTabsConfig, ConfigurationModelAdmin)
 admin.site.register(VideoUploadConfig, ConfigurationModelAdmin)
 admin.site.register(CourseOutlineRegenerate, CourseOutlineRegenerateAdmin)

cms/djangoapps/contentstore/helpers.py

@@ -29,6 +29,7 @@
 from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
 import openedx.core.djangoapps.content_staging.api as content_staging_api
 import openedx.core.djangoapps.content_tagging.api as content_tagging_api
+from openedx.core.djangoapps.content_staging.data import LIBRARY_SYNC_PURPOSE
 
 from .utils import reverse_course_url, reverse_library_url, reverse_usage_url
 
@@ -262,6 +263,37 @@ class StaticFileNotices:
     error_files: list[str] = Factory(list)
 
 
+def _insert_static_files_into_downstream_xblock(
+    downstream_xblock: XBlock, staged_content_id: int, request
+) -> StaticFileNotices:
+    """
+    Gets static files from staged content, and inserts them into the downstream XBlock.
+    """
+    static_files = content_staging_api.get_staged_content_static_files(staged_content_id)
+    notices, substitutions = _import_files_into_course(
+        course_key=downstream_xblock.context_key,
+        staged_content_id=staged_content_id,
+        static_files=static_files,
+        usage_key=downstream_xblock.scope_ids.usage_id,
+    )
+
+    # Rewrite the OLX's static asset references to point to the new
+    # locations for those assets. See _import_files_into_course for more
+    # info on why this is necessary.
+    store = modulestore()
+    if hasattr(downstream_xblock, "data") and substitutions:
+        data_with_substitutions = downstream_xblock.data
+        for old_static_ref, new_static_ref in substitutions.items():
+            data_with_substitutions = data_with_substitutions.replace(
+                old_static_ref,
+                new_static_ref,
+            )
+        downstream_xblock.data = data_with_substitutions
+        if store is not None:
+            store.update_item(downstream_xblock, request.user.id)
+    return notices
+
+
 def import_staged_content_from_user_clipboard(parent_key: UsageKey, request) -> tuple[XBlock | None, StaticFileNotices]:
     """
     Import a block (along with its children and any required static assets) from
@@ -299,31 +331,43 @@ def import_staged_content_from_user_clipboard(parent_key: UsageKey, request) ->
             tags=user_clipboard.content.tags,
         )
 
-        # Now handle static files that need to go into Files & Uploads.
-        static_files = content_staging_api.get_staged_content_static_files(user_clipboard.content.id)
-        notices, substitutions = _import_files_into_course(
-            course_key=parent_key.context_key,
-            staged_content_id=user_clipboard.content.id,
-            static_files=static_files,
-            usage_key=new_xblock.scope_ids.usage_id,
-        )
-
-        # Rewrite the OLX's static asset references to point to the new
-        # locations for those assets. See _import_files_into_course for more
-        # info on why this is necessary.
-        if hasattr(new_xblock, 'data') and substitutions:
-            data_with_substitutions = new_xblock.data
-            for old_static_ref, new_static_ref in substitutions.items():
-                data_with_substitutions = data_with_substitutions.replace(
-                    old_static_ref,
-                    new_static_ref,
-                )
-            new_xblock.data = data_with_substitutions
-            store.update_item(new_xblock, request.user.id)
+        notices = _insert_static_files_into_downstream_xblock(new_xblock, user_clipboard.content.id, request)
 
     return new_xblock, notices
 
 
+def import_static_assets_for_library_sync(downstream_xblock: XBlock, lib_block: XBlock, request) -> StaticFileNotices:
+    """
+    Import the static assets from the library xblock to the downstream xblock
+    through staged content. Also updates the OLX references to point to the new
+    locations of those assets in the downstream course.
+
+    Does not deal with permissions or REST stuff - do that before calling this.
+
+    Returns a summary of changes made to static files in the destination
+    course.
+    """
+    if not lib_block.runtime.get_block_assets(lib_block, fetch_asset_data=False):
+        return StaticFileNotices()
+    if not content_staging_api:
+        raise RuntimeError("The required content_staging app is not installed")
+    staged_content = content_staging_api.stage_xblock_temporarily(lib_block, request.user.id, LIBRARY_SYNC_PURPOSE)
+    if not staged_content:
+        # expired/error/loading
+        return StaticFileNotices()
+
+    store = modulestore()
+    try:
+        with store.bulk_operations(downstream_xblock.context_key):
+            # Now handle static files that need to go into Files & Uploads.
+            # If the required files already exist, nothing will happen besides updating the olx.
+            notices = _insert_static_files_into_downstream_xblock(downstream_xblock, staged_content.id, request)
+    finally:
+        staged_content.delete()
+
+    return notices
+
+
 def _fetch_and_set_upstream_link(
     copied_from_block: str,
     copied_from_version_num: int,
@@ -548,6 +592,9 @@ def _import_files_into_course(
             if result is True:
                 new_files.append(file_data_obj.filename)
                 substitutions.update(substitution_for_file)
+            elif substitution_for_file:
+                # substitutions need to be made because OLX references to these files need to be updated
+                substitutions.update(substitution_for_file)
             elif result is None:
                 pass  # This file already exists; no action needed.
             else:
@@ -618,8 +665,8 @@ def _import_file_into_course(
         contentstore().save(content)
         return True, {clipboard_file_path: f"static/{import_path}"}
     elif current_file.content_digest == file_data_obj.md5_hash:
-        # The file already exists and matches exactly, so no action is needed
-        return None, {}
+        # The file already exists and matches exactly, so no action is needed except substitutions
+        return None, {clipboard_file_path: f"static/{import_path}"}
     else:
         # There is a conflict with some other file that has the same name.
         return False, {}

cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py

@@ -0,0 +1,94 @@
+"""
+Management command to recreate upstream-dowstream links in PublishableEntityLink for course(s).
+
+This command can be run for all the courses or for given list of courses.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+
+from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import gettext as _
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.keys import CourseKey
+
+from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
+
+from ...tasks import create_or_update_upstream_links
+
+log = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Recreate links for course(s) in PublishableEntityLink table.
+
+    Examples:
+        # Recreate upstream links for two courses.
+        $ ./manage.py cms recreate_upstream_links --course course-v1:edX+DemoX.1+2014 \
+        --course course-v1:edX+DemoX.2+2015
+        # Force recreate upstream links for one or more courses including processed ones.
+        $ ./manage.py cms recreate_upstream_links --course course-v1:edX+DemoX.1+2014 \
+        --course course-v1:edX+DemoX.2+2015 --force
+        # Recreate upstream links for all courses.
+        $ ./manage.py cms recreate_upstream_links --all
+        # Force recreate links for all courses including completely processed ones.
+        $ ./manage.py cms recreate_upstream_links --all --force
+        # Delete all links and force recreate links for all courses
+        $ ./manage.py cms recreate_upstream_links --all --force --replace
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--course',
+            metavar=_('COURSE_KEY'),
+            action='append',
+            help=_('Recreate links for xblocks under given course keys. For eg. course-v1:edX+DemoX.1+2014'),
+            default=[],
+        )
+        parser.add_argument(
+            '--all',
+            action='store_true',
+            help=_(
+                'Recreate links for xblocks under all courses. NOTE: this can take long time depending'
+                ' on number of course and xblocks'
+            ),
+        )
+        parser.add_argument(
+            '--force',
+            action='store_true',
+            help=_('Recreate links even for completely processed courses.'),
+        )
+        parser.add_argument(
+            '--replace',
+            action='store_true',
+            help=_('Delete all and create links for given course(s).'),
+        )
+
+    def handle(self, *args, **options):
+        """
+        Handle command
+        """
+        courses = options['course']
+        should_process_all = options['all']
+        force = options['force']
+        replace = options['replace']
+        time_now = datetime.now(tz=timezone.utc)
+        if not courses and not should_process_all:
+            raise CommandError('Either --course or --all argument should be provided.')
+
+        if should_process_all and courses:
+            raise CommandError('Only one of --course or --all argument should be provided.')
+
+        if should_process_all:
+            courses = CourseOverview.get_all_course_keys()
+        for course in courses:
+            log.info(f"Start processing upstream->dowstream links in course: {course}")
+            try:
+                CourseKey.from_string(str(course))
+            except InvalidKeyError:
+                log.error(f"Invalid course key: {course}, skipping..")
+                continue
+            create_or_update_upstream_links.delay(str(course), force=force, replace=replace, created=time_now)