From 62e4738d2da5367a539c3676cc90530ac1fa4c0b Mon Sep 17 00:00:00 2001 From: Steve Rice Date: Thu, 26 Mar 2020 22:55:10 -0700 Subject: [PATCH] Multithread emoji download To improve emoji download performance, split the emoji work into multiple threads. --- lib/app.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/app.py b/lib/app.py index 282fa37..ce880ab 100644 --- a/lib/app.py +++ b/lib/app.py @@ -1,6 +1,7 @@ import os import subprocess from io import BytesIO +from multiprocessing.dummy import Pool from os.path import expanduser from pathlib import Path from typing import Dict @@ -171,14 +172,21 @@ def check_if_exists_map(self, args): def get_custom_emoji(self): data = self.slack.web_client.emoji_list() + work_pool = Pool(WORK_POOL_SIZE) + emoji_to_download = list( - filter(None, map(self.check_if_exists_map, data["emoji"].items()),) + filter( + None, + work_pool.imap_unordered( + self.check_if_exists_map, data["emoji"].items() + ), + ) ) num_emoji = len(emoji_to_download) list( tqdm( - map(self.update_emoji_map, emoji_to_download), + work_pool.imap_unordered(self.update_emoji_map, emoji_to_download), desc="Downloading Custom Emoji", unit="emoji", total=num_emoji,