Threading-Queue/image_multithreading_dowload.py at master · moondra2017/Threading-Queue · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

import os
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
import time
import queue
from threading import Thread


SAVE_DIR = r'C:\Users\Moondra\Desktop\TEMP\Puppy_threading'


def decorator_function(func):
    def wrapper(*args,**kwargs):
        session = requests.Session()
        retry = Retry(connect=0, backoff_factor=0.2)
        adapter = HTTPAdapter(max_retries=retry)
        session.mount('http://', adapter)
        session.mount('https://', adapter)

        return func(*args, session = session, **kwargs)
    return wrapper


#Using threading:
image_count = 0

#optional decorator_function
#@decorator_function
def download_image(SAVE_DIR,q, session = None):
        global image_count
        if not session:
                session = requests.Session()
        while not q.empty():

            try:

                    r = session.get(q.get(block = False))

            except (requests.exceptions.RequestException, UnicodeError) as e:
                print(e)
                image_count += 1
                q.task_done()
                continue

            image_count += 1
            q.task_done()

            print('image', image_count)
            with open(os.path.join(
                        SAVE_DIR, 'image_{}.jpg'.format(image_count)),
                        'wb') as f:

                f.write(r.content)


q =queue.Queue()
with open(r'C:\Users\Moondra\Desktop\puppies.txt', 'rt') as f:
    for i in range(200):
        line = f.readline()
        q.put(line.strip())
print(q.qsize())


threads = []
start = time.time()
for i in range(20):
     t = Thread(target = download_image,
                args = (SAVE_DIR,q))
     #t.setDaemon(True)
     threads.append(t)
     t.start()
q.join()

for t in threads:
    t.join()
    print(t.name, 'has joined')

end = time.time()
print('time taken: {:.4f}'.format(end - start))


#time taken: 7.4640
#time taken: 5.0860