feat: rework ExpectedPackages generation/management to add PieceInstances as owners to existing docs

Author: Julusian

meteor/server/api/__tests__/cleanup.test.ts

@@ -261,7 +261,10 @@ async function setDefaultDatatoDB(env: DefaultEnvironment, now: number) {
 		bucketId: null,
 		created: 0,
 		package: {} as any,
-		ingestSources: [] as any,
+		ingestSources: [],
+		playoutSources: {
+			pieceInstanceIds: [],
+		},
 	})
 	await ExpectedPackageWorkStatuses.insertAsync({
 		_id: getRandomId(),

meteor/server/migration/X_X_X.ts

@@ -142,6 +142,9 @@ export const addSteps = addMigrationSteps(CURRENT_SYSTEM_VERSION, [
 						},
 						created: pkg.created,
 						ingestSources: [ingestSource],
+						playoutSources: {
+							pieceInstanceIds: [],
+						},
 					} satisfies Complete<ExpectedPackageDB>)
 				}
 			}

meteor/server/publications/packageManager/expectedPackages/contentCache.ts

@@ -3,7 +3,7 @@ import { literal } from '@sofie-automation/corelib/dist/lib'
 import { MongoFieldSpecifierOnesStrict } from '@sofie-automation/corelib/dist/mongo'
 import { ExpectedPackageDB } from '@sofie-automation/corelib/dist/dataModel/ExpectedPackages'
 import { DBRundownPlaylist } from '@sofie-automation/corelib/dist/dataModel/RundownPlaylist'
-import { PieceInstance, PieceInstancePiece } from '@sofie-automation/corelib/dist/dataModel/PieceInstance'
+import { PieceInstance } from '@sofie-automation/corelib/dist/dataModel/PieceInstance'
 
 export type RundownPlaylistCompact = Pick<
 	DBRundownPlaylist,
@@ -16,16 +16,16 @@ export const rundownPlaylistFieldSpecifier = literal<MongoFieldSpecifierOnesStri
 	nextPartInfo: 1, // So that it invalidates when the next changes
 })
 
-export type PieceInstanceCompact = Pick<PieceInstance, '_id' | 'rundownId'> & {
-	piece: Pick<PieceInstancePiece, 'expectedPackages'>
-}
+export type PieceInstanceCompact = Pick<
+	PieceInstance,
+	'_id' | 'rundownId' | 'partInstanceId' | 'neededExpectedPackageIds'
+>
 
 export const pieceInstanceFieldsSpecifier = literal<MongoFieldSpecifierOnesStrict<PieceInstanceCompact>>({
 	_id: 1,
 	rundownId: 1,
-	piece: {
-		expectedPackages: 1,
-	},
+	partInstanceId: 1,
+	neededExpectedPackageIds: 1,
 })
 
 export type ExpectedPackageDBCompact = Pick<ExpectedPackageDB, '_id' | 'package'>

meteor/server/publications/packageManager/expectedPackages/contentObserver.ts

@@ -5,7 +5,6 @@ import {
 	ExpectedPackagesContentCache,
 	rundownPlaylistFieldSpecifier,
 	pieceInstanceFieldsSpecifier,
-	expectedPackageDBFieldsSpecifier,
 } from './contentCache'
 import { ExpectedPackages, PieceInstances, RundownPlaylists } from '../../../collections'
 import { ReactiveMongoObserverGroup, ReactiveMongoObserverGroupHandle } from '../../lib/observerGroup'
@@ -62,10 +61,7 @@ export class ExpectedPackagesContentObserver implements Meteor.LiveQueryHandle {
 				{
 					studioId: studioId,
 				},
-				cache.ExpectedPackages.link(),
-				{
-					projection: expectedPackageDBFieldsSpecifier,
-				}
+				cache.ExpectedPackages.link()
 			),
 
 			RundownPlaylists.observeChanges(

meteor/server/publications/packageManager/expectedPackages/generate.ts

@@ -1,6 +1,10 @@
-import { PackageContainerOnPackage, Accessor, AccessorOnPackage } from '@sofie-automation/blueprints-integration'
-import { getExpectedPackageIdForPieceInstance } from '@sofie-automation/corelib/dist/dataModel/ExpectedPackages'
-import { PeripheralDeviceId, ExpectedPackageId, PieceInstanceId } from '@sofie-automation/corelib/dist/dataModel/Ids'
+import {
+	PackageContainerOnPackage,
+	Accessor,
+	AccessorOnPackage,
+	ExpectedPackage,
+} from '@sofie-automation/blueprints-integration'
+import { PeripheralDeviceId, ExpectedPackageId } from '@sofie-automation/corelib/dist/dataModel/Ids'
 import { protectString, unprotectString } from '@sofie-automation/corelib/dist/protectedString'
 import {
 	PackageManagerExpectedPackage,
@@ -15,7 +19,7 @@ import { DBStudio, StudioLight, StudioPackageContainer } from '@sofie-automation
 import { clone, omit } from '../../../lib/tempLib'
 import { CustomPublishCollection } from '../../../lib/customPublication'
 import { logger } from '../../../logging'
-import { ExpectedPackagesContentCache } from './contentCache'
+import { ExpectedPackageDBCompact, ExpectedPackagesContentCache } from './contentCache'
 import type { StudioFields } from './publication'
 
 /**
@@ -59,17 +63,7 @@ export async function updateCollectionForExpectedPackageIds(
 			// Filter, keep only the routed mappings for this device:
 			if (filterPlayoutDeviceIds && !filterPlayoutDeviceIds.includes(deviceId)) continue
 
-			const routedPackage = generateExpectedPackageForDevice(
-				studio,
-				{
-					...packageDoc.package,
-					_id: unprotectString(packageDoc._id),
-				},
-				deviceId,
-				null,
-				Priorities.OTHER, // low priority
-				packageContainers
-			)
+			const routedPackage = generateExpectedPackageForDevice(studio, packageDoc, deviceId, packageContainers)
 
 			updatedDocIds.add(routedPackage._id)
 			collection.replace(routedPackage)
@@ -78,95 +72,15 @@ export async function updateCollectionForExpectedPackageIds(
 
 	// Remove all documents for an ExpectedPackage that was regenerated, and no update was issues
 	collection.remove((doc) => {
-		if (doc.pieceInstanceId) return false
-
-		if (missingExpectedPackageIds.has(protectString(doc.expectedPackage._id))) return true
-
-		if (updatedDocIds.has(doc._id) && !regenerateIds.has(protectString(doc.expectedPackage._id))) return true
-
-		return false
-	})
-}
-
-/**
- * Regenerate the output for the provided PieceInstance `regenerateIds`, updating the data in `collection` as needed
- * @param contentCache Cache of the database documents used
- * @param studio Minimal studio document
- * @param layerNameToDeviceIds Lookup table of package layers, to PeripheralDeviceIds the layer could be used with
- * @param collection Output collection of the publication
- * @param filterPlayoutDeviceIds PeripheralDeviceId filter applied to this publication
- * @param regenerateIds Ids of PieceInstance documents to be recalculated
- */
-export async function updateCollectionForPieceInstanceIds(
-	contentCache: ReadonlyDeep<ExpectedPackagesContentCache>,
-	studio: Pick<DBStudio, StudioFields>,
-	layerNameToDeviceIds: Map<string, PeripheralDeviceId[]>,
-	packageContainers: Record<string, StudioPackageContainer>,
-	collection: CustomPublishCollection<PackageManagerExpectedPackage>,
-	filterPlayoutDeviceIds: ReadonlyDeep<PeripheralDeviceId[]> | undefined,
-	regenerateIds: Set<PieceInstanceId>
-): Promise<void> {
-	const updatedDocIds = new Set<PackageManagerExpectedPackageId>()
-	const missingPieceInstanceIds = new Set<PieceInstanceId>()
-
-	for (const pieceInstanceId of regenerateIds) {
-		const pieceInstanceDoc = contentCache.PieceInstances.findOne(pieceInstanceId)
-		if (!pieceInstanceDoc) {
-			missingPieceInstanceIds.add(pieceInstanceId)
-			continue
-		}
-		if (!pieceInstanceDoc.piece?.expectedPackages) continue
-
-		pieceInstanceDoc.piece.expectedPackages.forEach((expectedPackage, i) => {
-			const sanitisedPackageId = getExpectedPackageIdForPieceInstance(
-				pieceInstanceId,
-				expectedPackage._id || '__unnamed' + i
-			)
-
-			// Map the expectedPackages onto their specified layer:
-			const allDeviceIds = new Set<PeripheralDeviceId>()
-			for (const layerName of expectedPackage.layers) {
-				const layerDeviceIds = layerNameToDeviceIds.get(layerName)
-				for (const deviceId of layerDeviceIds || []) {
-					allDeviceIds.add(deviceId)
-				}
-			}
-
-			for (const deviceId of allDeviceIds) {
-				// Filter, keep only the routed mappings for this device:
-				if (filterPlayoutDeviceIds && !filterPlayoutDeviceIds.includes(deviceId)) continue
-
-				const routedPackage = generateExpectedPackageForDevice(
-					studio,
-					{
-						...expectedPackage,
-						_id: unprotectString(sanitisedPackageId),
-					},
-					deviceId,
-					pieceInstanceId,
-					Priorities.OTHER, // low priority
-					packageContainers
-				)
-
-				updatedDocIds.add(routedPackage._id)
-				collection.replace(routedPackage)
-			}
-		})
-	}
-
-	// Remove all documents for an ExpectedPackage that was regenerated, and no update was issues
-	collection.remove((doc) => {
-		if (!doc.pieceInstanceId) return false
-
-		if (missingPieceInstanceIds.has(doc.pieceInstanceId)) return true
+		if (missingExpectedPackageIds.has(doc.expectedPackage._id)) return true
 
-		if (updatedDocIds.has(doc._id) && !regenerateIds.has(doc.pieceInstanceId)) return true
+		if (updatedDocIds.has(doc._id) && !regenerateIds.has(doc.expectedPackage._id)) return true
 
 		return false
 	})
 }
 
-enum Priorities {
+export enum ExpectedPackagePriorities {
 	// Lower priorities are done first
 
 	/** Highest priority */
@@ -181,16 +95,14 @@ function generateExpectedPackageForDevice(
 		StudioLight,
 		'_id' | 'packageContainersWithOverrides' | 'previewContainerIds' | 'thumbnailContainerIds'
 	>,
-	expectedPackage: PackageManagerExpectedPackageBase,
+	expectedPackage: ExpectedPackageDBCompact,
 	deviceId: PeripheralDeviceId,
-	pieceInstanceId: PieceInstanceId | null,
-	priority: Priorities,
 	packageContainers: Record<string, StudioPackageContainer>
 ): PackageManagerExpectedPackage {
 	// Lookup Package sources:
 	const combinedSources: PackageContainerOnPackage[] = []
 
-	for (const packageSource of expectedPackage.sources) {
+	for (const packageSource of expectedPackage.package.sources) {
 		const lookedUpSource = packageContainers[packageSource.containerId]
 		if (lookedUpSource) {
 			combinedSources.push(calculateCombinedSource(packageSource, lookedUpSource))
@@ -208,27 +120,27 @@ function generateExpectedPackageForDevice(
 	}
 
 	// Lookup Package targets:
-	const combinedTargets = calculateCombinedTargets(expectedPackage, deviceId, packageContainers)
+	const combinedTargets = calculateCombinedTargets(expectedPackage.package, deviceId, packageContainers)
 
-	if (!combinedSources.length && expectedPackage.sources.length !== 0) {
+	if (!combinedSources.length && expectedPackage.package.sources.length !== 0) {
 		logger.warn(`Pub.expectedPackagesForDevice: No sources found for "${expectedPackage._id}"`)
 	}
 	if (!combinedTargets.length) {
 		logger.warn(`Pub.expectedPackagesForDevice: No targets found for "${expectedPackage._id}"`)
 	}
-	const packageSideEffect = getSideEffect(expectedPackage, studio)
+	const packageSideEffect = getSideEffect(expectedPackage.package, studio)
 
 	return {
-		_id: protectString(`${expectedPackage._id}_${deviceId}_${pieceInstanceId}`),
+		_id: protectString(`${expectedPackage._id}_${deviceId}`),
 		expectedPackage: {
-			...expectedPackage,
+			...expectedPackage.package,
+			_id: expectedPackage._id,
 			sideEffect: packageSideEffect,
 		},
 		sources: combinedSources,
 		targets: combinedTargets,
-		priority: priority,
+		priority: ExpectedPackagePriorities.OTHER, // This gets overriden later if needed
 		playoutDeviceId: deviceId,
-		pieceInstanceId,
 	}
 }
 
@@ -265,7 +177,7 @@ function calculateCombinedSource(
 	return combinedSource
 }
 function calculateCombinedTargets(
-	expectedPackage: PackageManagerExpectedPackageBase,
+	expectedPackage: ReadonlyDeep<ExpectedPackage.Base>,
 	deviceId: PeripheralDeviceId,
 	packageContainers: Record<string, StudioPackageContainer>
 ): PackageContainerOnPackage[] {

meteor/server/publications/packageManager/expectedPackages/publication.ts

@@ -23,7 +23,7 @@ import { PackageManagerExpectedPackage } from '@sofie-automation/shared-lib/dist
 import { ExpectedPackagesContentObserver } from './contentObserver'
 import { createReactiveContentCache, ExpectedPackagesContentCache } from './contentCache'
 import { buildMappingsToDeviceIdMap } from './util'
-import { updateCollectionForExpectedPackageIds, updateCollectionForPieceInstanceIds } from './generate'
+import { ExpectedPackagePriorities, updateCollectionForExpectedPackageIds } from './generate'
 import {
 	PeripheralDevicePubSub,
 	PeripheralDevicePubSubCollectionsNames,
@@ -161,16 +161,13 @@ async function manipulateExpectedPackagesPublicationData(
 	}
 
 	let regenerateExpectedPackageIds: Set<ExpectedPackageId>
-	let regeneratePieceInstanceIds: Set<PieceInstanceId>
 	if (invalidateAllItems) {
-		// force every piece to be regenerated
+		// force every package to be regenerated
 		collection.remove(null)
 		regenerateExpectedPackageIds = new Set(state.contentCache.ExpectedPackages.find({}).map((p) => p._id))
-		regeneratePieceInstanceIds = new Set(state.contentCache.PieceInstances.find({}).map((p) => p._id))
 	} else {
 		// only regenerate the reported changes
 		regenerateExpectedPackageIds = new Set(updateProps.invalidateExpectedPackageIds)
-		regeneratePieceInstanceIds = new Set(updateProps.invalidatePieceInstanceIds)
 	}
 
 	await updateCollectionForExpectedPackageIds(
@@ -182,15 +179,49 @@ async function manipulateExpectedPackagesPublicationData(
 		args.filterPlayoutDeviceIds,
 		regenerateExpectedPackageIds
 	)
-	await updateCollectionForPieceInstanceIds(
-		state.contentCache,
-		state.studio,
-		state.layerNameToDeviceIds,
-		state.packageContainers,
-		collection,
-		args.filterPlayoutDeviceIds,
-		regeneratePieceInstanceIds
-	)
+
+	// Ensure the priorities are correct for the packages
+	// We can do this as a post-step, as it means we can generate the packages solely based on the content
+	// If one gets regenerated, its priority will be reset to OTHER. But as it has already changed, this fixup is 'free'
+	// For those not regenerated, we can set the priority to the correct value if it has changed, without any deeper checks
+	updatePackagePriorities(state.contentCache, collection)
+}
+
+function updatePackagePriorities(
+	contentCache: ReadonlyDeep<ExpectedPackagesContentCache>,
+	collection: CustomPublishCollection<PackageManagerExpectedPackage>
+) {
+	const highPriorityPackages = new Map<ExpectedPackageId, ExpectedPackagePriorities>()
+
+	// Compile the map of the expected priority of each package
+	const knownPieceInstances = contentCache.PieceInstances.find({})
+	const playlist = contentCache.RundownPlaylists.findOne({})
+	const currentPartInstanceId = playlist?.currentPartInfo?.partInstanceId
+	for (const pieceInstance of knownPieceInstances) {
+		const packageIds = pieceInstance.neededExpectedPackageIds
+		if (!packageIds) continue
+
+		const packagePriority =
+			pieceInstance.partInstanceId === currentPartInstanceId
+				? ExpectedPackagePriorities.PLAYOUT_CURRENT
+				: ExpectedPackagePriorities.PLAYOUT_NEXT
+
+		for (const packageId of packageIds) {
+			const existingPriority = highPriorityPackages.get(packageId) ?? ExpectedPackagePriorities.OTHER
+			highPriorityPackages.set(packageId, Math.min(existingPriority, packagePriority))
+		}
+	}
+
+	// Iterate through and update each package
+	collection.updateAll((pkg) => {
+		const expectedPriority = highPriorityPackages.get(pkg.expectedPackage._id) ?? ExpectedPackagePriorities.OTHER
+		if (pkg.priority === expectedPriority) return false
+
+		return {
+			...pkg,
+			priority: expectedPriority,
+		}
+	})
 }
 
 meteorCustomPublish(