Not Collecting All Space Consuming Items

[chuccck]

The script says it has found ~47000 items filtering for space consuming. When I tell it to add to new album the first time, it adds ~19,850 into a new album. That's fine, just part of the 20k limit. I run it a second time with the first new album excluded, then it only adds ~5K items to the 2nd new album. When I run it a 3rd time with the first two albums excluded, I get 0 items. Why is it saying ~47k items if it can only add ~25k items to albums?

[djraval]

I wanted to achieve something similar so got came up with this, hope this helps you.
Do note that you might need to adjust the mediaInfo size, i've set mine to 20MB.

async function addSpaceConsumingItemsToAlbums() {
  const baseAlbumName = 'Space Consuming All';
  const albumLimit = 20000;
  let allItems = [];
  let nextPageId = null;

  console.log('Fetching all items from the library...');

  do {
    try {
      const page = await gptkApi.getItemsByUploadedDate(nextPageId);
      if (page && Array.isArray(page.items)) {
        allItems = allItems.concat(page.items);
        console.log(`Fetched ${page.items.length} items. Total items: ${allItems.length}`);
      } else {
        console.log('Received a page with no items.');
      }
      nextPageId = page?.nextPageId || null;
    } catch (error) {
      console.error('Error occurred while fetching items:', error);
    }

    await new Promise(resolve => setTimeout(resolve, 1000));
  } while (nextPageId);

  console.log(`Finished fetching. Total items in library: ${allItems.length}`);

  console.log('Processing items to find space-consuming ones...');
  let spaceConsumingItems = [];
  const batchSize = 5000; // Adjust based on API limitations

  for (let i = 0; i < allItems.length; i += batchSize) {
    const batch = allItems.slice(i, i + batchSize);
    const mediaKeys = batch.map(item => item.mediaKey).filter(Boolean);
    try {
      if (mediaKeys.length > 0) {
        const mediaInfo = await gptkApi.getBatchMediaInfo(mediaKeys);
        const spaceConsumingBatchItems = batch.filter((item, index) => 
          mediaInfo[index] && mediaInfo[index].size > 20 * 1024 * 1024 // 20MB
        );
        spaceConsumingItems = spaceConsumingItems.concat(spaceConsumingBatchItems);
      }
      console.log(`Processed ${i + batch.length}/${allItems.length} items. Space-consuming items found: ${spaceConsumingItems.length}`);
    } catch (error) {
      console.error('Error occurred while getting media info:', error);
    }
    await new Promise(resolve => setTimeout(resolve, 500));
  }

  console.log(`Finished processing. Total space-consuming items found: ${spaceConsumingItems.length}`);

  if (spaceConsumingItems.length > 0) {
    const numberOfAlbums = Math.ceil(spaceConsumingItems.length / albumLimit);
    const albumNames = Array.from({ length: numberOfAlbums }, (_, i) => `${baseAlbumName} (${i + 1}/${numberOfAlbums})`);

    console.log(`Creating ${numberOfAlbums} album(s) due to the 20,000 item limit per album.`);

    for (let albumIndex = 0; albumIndex < numberOfAlbums; albumIndex++) {
      const albumName = albumNames[albumIndex];
      const startIndex = albumIndex * albumLimit;
      const endIndex = Math.min((albumIndex + 1) * albumLimit, spaceConsumingItems.length);
      const albumItems = spaceConsumingItems.slice(startIndex, endIndex);

      try {
        console.log(`Creating new album: ${albumName}`);
        const albumId = await gptkApi.createAlbum(albumName);
        console.log(`Album created with ID: ${albumId}`);

        for (let i = 0; i < albumItems.length; i += batchSize) {
          const batch = albumItems.slice(i, i + batchSize);
          const mediaKeys = batch.map(item => item.mediaKey).filter(Boolean);
          if (mediaKeys.length > 0) {
            await gptkApi.addItemsToAlbum(mediaKeys, albumId);
            console.log(`Added ${mediaKeys.length} items to the album "${albumName}". Progress: ${i + batch.length}/${albumItems.length}`);
          }
        }

        console.log(`Successfully added all ${albumItems.length} items to the album "${albumName}"`);
      } catch (error) {
        console.error(`Error occurred while creating album "${albumName}" or adding items:`, error);
      }
    }

    console.log(`Finished adding all ${spaceConsumingItems.length} space-consuming items to ${numberOfAlbums} album(s).`);
  } else {
    console.log('No space-consuming items found in the library.');
  }
}

// Run the function
addSpaceConsumingItemsToAlbums();