fix: extra_query_params

Author: CatchZeng

.vscode/settings.json

@@ -1,10 +1,14 @@
 {
-    "python.linting.pylintEnabled": true,
-    "python.linting.enabled": true,
-    "cSpell.words": [
-        "adlt",
-        "imap",
-        "murl",
-        "posixpath"
-    ]
-}
+  "python.linting.pylintEnabled": true,
+  "python.linting.enabled": true,
+  "cSpell.words": [
+    "adlt",
+    "filterui",
+    "imap",
+    "iusc",
+    "mbot",
+    "murl",
+    "posixpath"
+  ],
+  "python.formatting.provider": "yapf"
+}

bing_images/bing.py

@@ -26,7 +26,7 @@ def fetch_image_urls(
     keywords = query
     if len(file_type) > 0:
         keywords = query + " " + file_type
-    urls = crawl_image_urls(keywords, filters, limit, extra_query_params)
+    urls = crawl_image_urls(keywords, filters, limit, extra_query_params=extra_query_params)
     for url in urls:
         if isValidURL(url, file_type) and url not in result:
             result.append(url)
@@ -57,7 +57,7 @@ def download_images(
 
     # Fetch more image URLs to avoid some images are invalid.
     max_number = math.ceil(limit*1.5)
-    urls = fetch_image_urls(query, max_number, file_type, filters, extra_query_params)
+    urls = fetch_image_urls(query, max_number, file_type, filters, extra_query_params=extra_query_params)
     entries = get_image_entries(urls, image_dir)
 
     print("Downloading images")
@@ -131,4 +131,5 @@ def download_image_with_thread(entry):
                     output_dir="/Users/catchzeng/Desktop/cat",
                     pool_size=10,
                     file_type="png",
-                    force_replace=True)
+                    force_replace=True,
+                    extra_query_params='&first=100')

bing_images/crawler.py

@@ -7,7 +7,7 @@
 BASE_URL = "https://www.bing.com/images/search?"
 
 
-def gen_query_url(keywords, filters, extra_query_params =''):
+def gen_query_url(keywords, filters, extra_query_params=''):
     keywords_str = "&q=" + quote(keywords)
     query_url = BASE_URL + keywords_str
     if len(filters) > 0:
@@ -44,7 +44,7 @@ def image_url_from_webpage(driver, max_number=10000):
     return image_urls
 
 
-def crawl_image_urls(keywords, filters, max_number=10000, proxy=None, proxy_type="http", extra_query_params =''):
+def crawl_image_urls(keywords, filters, max_number=10000, proxy=None, proxy_type="http", extra_query_params=''):
     chrome_path = shutil.which("chromedriver")
     chrome_path = "./bin/chromedriver" if chrome_path is None else chrome_path
     chrome_options = webdriver.ChromeOptions()
@@ -53,7 +53,7 @@ def crawl_image_urls(keywords, filters, max_number=10000, proxy=None, proxy_type
             "--proxy-server={}://{}".format(proxy_type, proxy))
     driver = webdriver.Chrome(chrome_path, chrome_options=chrome_options)
 
-    query_url = gen_query_url(keywords, filters, extra_query_params)
+    query_url = gen_query_url(keywords, filters, extra_query_params=extra_query_params)
     driver.set_window_size(1920, 1080)
     driver.get(query_url)
     image_urls = image_url_from_webpage(driver, max_number)
@@ -72,6 +72,6 @@ def crawl_image_urls(keywords, filters, max_number=10000, proxy=None, proxy_type
 
 if __name__ == '__main__':
     images = crawl_image_urls(
-        "mbot png", "+filterui:aspect-square", max_number=10)
+        "cat png", "+filterui:aspect-square", max_number=10)
     for i in images:
         print(i+"\n")

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = bing_images
-version = 0.2.1
+version = 0.2.2
 author = CatchZeng
 author_email = catchzenghh@gmail.com
 description = Python library to fetch image urls and download using multithreading from Bing.com.