Skip to content

Commit d513d97

Browse files
fantom-voronezhfantom-voronezh
authored
update lesson #06 nb
1 parent 6fb387d commit d513d97

File tree

1 file changed

+217
-10
lines changed

1 file changed

+217
-10
lines changed

‎lesson-06/class_06.ipynb‎

Lines changed: 217 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1015,12 +1015,15 @@
10151015
},
10161016
{
10171017
"cell_type": "code",
1018-
"execution_count": 1,
1018+
"execution_count": 3,
10191019
"id": "0314a21e-9661-4f7b-948b-6f856f3f65b5",
10201020
"metadata": {},
10211021
"outputs": [],
10221022
"source": [
1023-
"from urllib.request import urlopen"
1023+
"import threading\n",
1024+
"import time\n",
1025+
"from urllib.request import urlopen\n",
1026+
"import queue"
10241027
]
10251028
},
10261029
{
@@ -1159,7 +1162,7 @@
11591162
},
11601163
{
11611164
"cell_type": "code",
1162-
"execution_count": 33,
1165+
"execution_count": 4,
11631166
"id": "f3097b0c-3f85-4854-887b-df897e2e5b65",
11641167
"metadata": {},
11651168
"outputs": [
@@ -1171,12 +1174,12 @@
11711174
" Fetcher[0]\n",
11721175
"start Fetcher[2]\n",
11731176
"start Fetcher[3]\n",
1174-
"finish cnt=6 Fetcher[1] 3.8348538875579834\n",
1175-
"finish cnt=6 Fetcher[2] 3.9229462146759033\n",
1176-
"finish cnt=4 Fetcher[0] 3.9448299407958984\n",
1177-
"finish cnt=4 Fetcher[3] 5.434904098510742\n",
1178-
"CPU times: user 66.6 ms, sys: 29.9 ms, total: 96.5 ms\n",
1179-
"Wall time: 5.44 s\n"
1177+
"finish cnt=5finish cnt=5 Fetcher[3] 5.146667003631592\n",
1178+
" Fetcher[2] 5.1464009284973145\n",
1179+
"finish cnt=5 Fetcher[0] 5.158473014831543\n",
1180+
"finish cnt=5 Fetcher[1] 5.3739283084869385\n",
1181+
"CPU times: user 530 ms, sys: 40.5 ms, total: 571 ms\n",
1182+
"Wall time: 5.38 s\n"
11801183
]
11811184
}
11821185
],
@@ -1211,7 +1214,6 @@
12111214
" print(f\"finish {cnt=}\", threading.current_thread().name, time.time() - t1)\n",
12121215
"\n",
12131216
"\n",
1214-
"\n",
12151217
"def run():\n",
12161218
" que = queue.Queue()\n",
12171219
" for url in URLS:\n",
@@ -1245,6 +1247,211 @@
12451247
"metadata": {},
12461248
"outputs": [],
12471249
"source": []
1250+
},
1251+
{
1252+
"cell_type": "code",
1253+
"execution_count": null,
1254+
"id": "4aaf9d90-c4a1-424b-a99b-e35bd361f591",
1255+
"metadata": {},
1256+
"outputs": [],
1257+
"source": []
1258+
},
1259+
{
1260+
"cell_type": "code",
1261+
"execution_count": 10,
1262+
"id": "c6cd9155-21fd-4616-ac15-b3fd85eb8de0",
1263+
"metadata": {},
1264+
"outputs": [
1265+
{
1266+
"name": "stdout",
1267+
"output_type": "stream",
1268+
"text": [
1269+
"start Fetcher[0]\n",
1270+
"start Fetcher[1]\n",
1271+
"start Fetcher[2]\n",
1272+
"start Fetcher[3]\n",
1273+
"finish cnt=5finish cnt=5 Fetcher[2] 5.020750045776367\n",
1274+
"finish cnt=5 Fetcher[1] 5.021289825439453\n",
1275+
"finish cnt=5 Fetcher[0] 5.021651983261108\n",
1276+
" Fetcher[3] 5.019881963729858\n",
1277+
"CPU times: user 4.33 ms, sys: 3.33 ms, total: 7.66 ms\n",
1278+
"Wall time: 5.02 s\n"
1279+
]
1280+
}
1281+
],
1282+
"source": [
1283+
"%%time\n",
1284+
"\n",
1285+
"N = 20\n",
1286+
"URL = \"https://www.python.org/\"\n",
1287+
"URLS = [URL] * N\n",
1288+
"N_THREADS = 4\n",
1289+
"\n",
1290+
"\n",
1291+
"def fetch_url(url, sem):\n",
1292+
" with sem:\n",
1293+
" return urlopen(url)\n",
1294+
" #time.sleep(1)\n",
1295+
"\n",
1296+
"\n",
1297+
"def fetch_batch_worker(que, sem):\n",
1298+
" print(\"start\", threading.current_thread().name)\n",
1299+
" t1 = time.time()\n",
1300+
"\n",
1301+
" cnt = 0\n",
1302+
" \n",
1303+
" while True:\n",
1304+
" url = que.get()\n",
1305+
" if url is None:\n",
1306+
" que.put(url)\n",
1307+
" break\n",
1308+
"\n",
1309+
" fetch_url(url, sem)\n",
1310+
" cnt += 1\n",
1311+
"\n",
1312+
" print(f\"finish {cnt=}\", threading.current_thread().name, time.time() - t1)\n",
1313+
"\n",
1314+
"\n",
1315+
"def run():\n",
1316+
" que = queue.Queue()\n",
1317+
" for url in URLS:\n",
1318+
" que.put(url)\n",
1319+
"\n",
1320+
" que.put(None)\n",
1321+
"\n",
1322+
" sem = threading.Semaphore(4)\n",
1323+
"\n",
1324+
" threads = [\n",
1325+
" threading.Thread(\n",
1326+
" target=fetch_batch_worker,\n",
1327+
" args=(que, sem),\n",
1328+
" name=f\"Fetcher[{i}]\",\n",
1329+
" )\n",
1330+
" for i in range(N_THREADS)\n",
1331+
" ]\n",
1332+
" \n",
1333+
" for th in threads:\n",
1334+
" th.start()\n",
1335+
" \n",
1336+
" for th in threads:\n",
1337+
" th.join()\n",
1338+
"\n",
1339+
"\n",
1340+
"run()"
1341+
]
1342+
},
1343+
{
1344+
"cell_type": "code",
1345+
"execution_count": null,
1346+
"id": "e369e1fe-a12e-43d3-96cf-aa359fc8bfa2",
1347+
"metadata": {},
1348+
"outputs": [],
1349+
"source": []
1350+
},
1351+
{
1352+
"cell_type": "code",
1353+
"execution_count": null,
1354+
"id": "278f57ac-c55e-41f2-a525-1971da18e806",
1355+
"metadata": {},
1356+
"outputs": [],
1357+
"source": []
1358+
},
1359+
{
1360+
"cell_type": "code",
1361+
"execution_count": 14,
1362+
"id": "c3fa3f10-fd8c-48cf-9846-0b5ed2366caa",
1363+
"metadata": {},
1364+
"outputs": [
1365+
{
1366+
"name": "stdout",
1367+
"output_type": "stream",
1368+
"text": [
1369+
"start Fetcher[0]\n",
1370+
"start Fetcher[1]\n",
1371+
"process_error in threading.current_thread().name='Fetcher[0]' (_thread._ExceptHookArgs(exc_type=<class 'Exception'>, exc_value=Exception('Wrong cnt=2'), exc_traceback=<traceback object at 0x1118badc0>, thread=<Thread(Fetcher[0], started 13170143232)>),) {}\n",
1372+
"process_error in threading.current_thread().name='Fetcher[1]' (_thread._ExceptHookArgs(exc_type=<class 'Exception'>, exc_value=Exception('Wrong cnt=2'), exc_traceback=<traceback object at 0x11183bd80>, thread=<Thread(Fetcher[1], started 13186969600)>),) {}\n",
1373+
"OK\n",
1374+
"CPU times: user 14.3 ms, sys: 7.01 ms, total: 21.3 ms\n",
1375+
"Wall time: 1.53 s\n"
1376+
]
1377+
}
1378+
],
1379+
"source": [
1380+
"%%time\n",
1381+
"\n",
1382+
"N = 20\n",
1383+
"URL = \"https://www.python.org/\"\n",
1384+
"URLS = [URL] * N\n",
1385+
"N_THREADS = 2\n",
1386+
"\n",
1387+
"\n",
1388+
"def process_error(*args, **kwargs):\n",
1389+
" print(f\"process_error in {threading.current_thread().name=}\", args, kwargs)\n",
1390+
"\n",
1391+
"\n",
1392+
"threading.excepthook = process_error\n",
1393+
"\n",
1394+
"\n",
1395+
"def fetch_url(url):\n",
1396+
" return urlopen(url)\n",
1397+
"\n",
1398+
"\n",
1399+
"def fetch_batch_worker(que):\n",
1400+
" print(\"start\", threading.current_thread().name)\n",
1401+
" t1 = time.time()\n",
1402+
"\n",
1403+
" cnt = 0\n",
1404+
" \n",
1405+
" while True:\n",
1406+
" url = que.get()\n",
1407+
" if url is None:\n",
1408+
" que.put(url)\n",
1409+
" break\n",
1410+
"\n",
1411+
" fetch_url(url)\n",
1412+
" cnt += 1\n",
1413+
"\n",
1414+
" if cnt == 2:\n",
1415+
" raise Exception(\"Wrong cnt=2\")\n",
1416+
"\n",
1417+
" print(f\"finish {cnt=}\", threading.current_thread().name, time.time() - t1)\n",
1418+
"\n",
1419+
"\n",
1420+
"def run():\n",
1421+
" que = queue.Queue()\n",
1422+
" for url in URLS:\n",
1423+
" que.put(url)\n",
1424+
"\n",
1425+
" que.put(None)\n",
1426+
"\n",
1427+
" threads = [\n",
1428+
" threading.Thread(\n",
1429+
" target=fetch_batch_worker,\n",
1430+
" args=(que,),\n",
1431+
" name=f\"Fetcher[{i}]\",\n",
1432+
" )\n",
1433+
" for i in range(N_THREADS)\n",
1434+
" ]\n",
1435+
" \n",
1436+
" for th in threads:\n",
1437+
" th.start()\n",
1438+
" \n",
1439+
" for th in threads:\n",
1440+
" th.join()\n",
1441+
"\n",
1442+
"\n",
1443+
"run()\n",
1444+
"\n",
1445+
"print(\"OK\")"
1446+
]
1447+
},
1448+
{
1449+
"cell_type": "code",
1450+
"execution_count": null,
1451+
"id": "efd50394-0f37-4ac7-b2bb-89856d41004b",
1452+
"metadata": {},
1453+
"outputs": [],
1454+
"source": []
12481455
}
12491456
],
12501457
"metadata": {

0 commit comments

Comments
 (0)