Download latest driver version even if browser version is unknown (#260)

* Optimization: define env var GH_TOKEN for all jobs

* install stable chromium on mac

* Tests: adopt edge tests for CI.

* [#254] softly download latest webdriver version...

even if browser version is UNKNOWN.

* CI: add tests on windows github actions

* [#259] fix duplicating logs when "latest" version's found

* up version to 3.5.2 and update changelog

Author: aleksandr-kotlyar

.github/workflows/test.yml

@@ -9,10 +9,12 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-latest, macos-latest ]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
         python-version: [ '3.6', '3.10' ]
 
     steps:
@@ -48,13 +50,13 @@ jobs:
         shell: powershell
         run: |
           choco install chromium opera --no-progress -y --force
-          .github\set_win_reg_keys.ps1
 
       - name: Install browsers on MacOS
         if: startsWith(runner.os, 'macOS')
         run: |
+          brew tap domt4/chromium
           brew update
-          brew install --cask chromium opera
+          brew install --cask mac-chromium opera
 
       - name: Install Python dependencies
         run: |
@@ -64,24 +66,18 @@ jobs:
 
       - name: Run tests on Linux (with xvfb)
         if: runner.os == 'Linux'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         uses: GabrielBB/xvfb-action@v1.0
         with:
           run: |
             pipenv run py.test -sv --cov-config .coveragerc --cov-report xml --cov-report term:skip-covered --cov=webdriver_manager --tb=short tests/
 
       - name: Run tests on Windows (without xvfb)
         if: runner.os == 'Windows'
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           pipenv run py.test -sv --cov-config .coveragerc --cov-report xml --cov-report term:skip-covered --cov=webdriver_manager --tb=short tests/
 
       - name: Run tests on MacOS (without xvfb)
         if: startsWith(runner.os, 'macOS')
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           pipenv run py.test -sv --cov-config .coveragerc --cov-report xml --cov-report term:skip-covered --cov=webdriver_manager --tb=short tests/
 

CHANGELOG.md

@@ -2,9 +2,16 @@
 
 ---
 ## 3.5.2
-### SSL
-- Feature: now you can disable SSL verification by setting `os.environ['WDM_SSL_VERIFY']='0'` in case if you have troubles with SSL Certificates or SSL Certificate Chain (like in issues
+### Features
+- SSL verification can be disabled by setting `os.environ['WDM_SSL_VERIFY']='0'` in case if you have troubles with SSL Certificates or SSL Certificate Chain (like in issues
   [#219](https://github.com/SergeyPirogov/webdriver_manager/issues/219), [#226](https://github.com/SergeyPirogov/webdriver_manager/issues/226))
+### Fixes
+- Log duplication ([#259](https://github.com/SergeyPirogov/webdriver_manager/issues/259))
+- Failed to Download the Edge driver for particular Version ([#251](https://github.com/SergeyPirogov/webdriver_manager/issues/251))
+- WDM_LOG_LEVEL not work ([#255](https://github.com/SergeyPirogov/webdriver_manager/issues/255))
+### Improvements
+- Softly download latest webdriver version even if browser version is unknown. ([#254](https://github.com/SergeyPirogov/webdriver_manager/issues/254), also fixes [#251](https://github.com/SergeyPirogov/webdriver_manager/issues/251))
+- Speed up when using "latest" version ([#259](https://github.com/SergeyPirogov/webdriver_manager/issues/259))
 ---
 ## 3.5.1
 ### IEDriver

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.5.1
+current_version = 3.5.2
 commit = True
 tag = True
 

setup.py

@@ -23,7 +23,7 @@
     long_description_content_type="text/markdown",
     packages=setuptools.find_packages(exclude=['tests']),
     include_package_data=True,
-    version='3.5.1',
+    version='3.5.2',
     description=('Library provides the way to automatically manage drivers for different browsers'),
     author='Sergey Pirogov',
     author_email='automationremarks@gmail.com',

tests/test_edge_driver.py

@@ -45,19 +45,6 @@ def test_edge_manager_with_wrong_version():
            ) in ex.value.args[0]
 
 
-@pytest.mark.parametrize('os_type', ['win32', 'win64', 'linux64', 'mac64'])
-def test_can_download_edge_driver(os_type):
-    path = EdgeChromiumDriverManager(os_type=os_type).install()
-    assert os.path.exists(path)
-
-
-@pytest.mark.parametrize('os_type', ['win32', 'win64', 'mac64', 'linux64'])
-def test_can_get_edge_driver_from_cache(os_type):
-    EdgeChromiumDriverManager(os_type=os_type).install()
-    driver_path = EdgeChromiumDriverManager(os_type=os_type).install()
-    assert os.path.exists(driver_path)
-
-
 @pytest.mark.parametrize('os_type', ['win32', 'win64', 'mac64', 'linux64'])
 @pytest.mark.parametrize('specific_version', ['86.0.600.0'])
 def test_edge_with_specific_version(os_type, specific_version):
@@ -66,3 +53,17 @@ def test_edge_with_specific_version(os_type, specific_version):
         os_type=os_type,
     ).install()
     assert os.path.exists(bin_path)
+
+
+@pytest.mark.parametrize('os_type', ['win32', 'win64', 'mac64', 'linux64'])
+@pytest.mark.parametrize('specific_version', ['87.0.637.0'])
+def test_can_get_edge_driver_from_cache(os_type, specific_version):
+    EdgeChromiumDriverManager(
+        version=specific_version,
+        os_type=os_type,
+    ).install()
+    driver_path = EdgeChromiumDriverManager(
+        version=specific_version,
+        os_type=os_type
+    ).install()
+    assert os.path.exists(driver_path)

webdriver_manager/__init__.py

@@ -1 +1 @@
-__version__ = '3.5.1'
+__version__ = '3.5.2'

webdriver_manager/chrome.py

@@ -3,7 +3,6 @@
 
 from webdriver_manager import utils
 from webdriver_manager.driver import ChromeDriver
-from webdriver_manager.logger import log
 from webdriver_manager.manager import DriverManager
 from webdriver_manager.utils import ChromeType
 
@@ -30,7 +29,6 @@ def __init__(self, version="latest",
                                    chrome_type=chrome_type)
 
     def install(self):
-        log(f"Current {self.driver.chrome_type} version is {self.driver.browser_version}", first_line=True)
         driver_path = self._get_driver_path(self.driver)
 
         os.chmod(driver_path, 0o755)

webdriver_manager/driver.py

@@ -52,21 +52,28 @@ def __init__(self, name, version, os_type, url, latest_release_url,
         super(ChromeDriver, self).__init__(name, version, os_type, url,
                                            latest_release_url)
         self.chrome_type = chrome_type
-        self.browser_version = get_browser_version_from_os(chrome_type)
+        self.browser_version = ""
 
     def get_os_type(self):
         if "win" in super().get_os_type():
             return "win32"
         return super().get_os_type()
 
     def get_latest_release_version(self):
-        log(f"Get LATEST driver version for {self.browser_version}")
+        self.browser_version = get_browser_version_from_os(self.chrome_type)
+        log(f"Get LATEST {self._name} version for {self.browser_version} {self.chrome_type}")
+        latest_release_url = (
+            f"{self._latest_release_url}_{self.browser_version}"
+            if self.browser_version != "UNKNOWN"
+            else self._latest_release_url
+        )
         resp = requests.get(
-            url=f"{self._latest_release_url}_{self.browser_version}",
+            url=latest_release_url,
             verify=self.ssl_verify
         )
         validate_response(resp)
-        return resp.text.rstrip()
+        self._version = resp.text.rstrip()
+        return self._version
 
 
 class GeckoDriver(Driver):
@@ -87,7 +94,7 @@ def __init__(
             latest_release_url,
         )
         self._mozila_release_tag = mozila_release_tag
-        self.browser_version = firefox_version()
+        self.browser_version = ""
         self._os_token = os.getenv("GH_TOKEN", None)
         self.auth_header = (
             {'Authorization': f'token {self._os_token}'}
@@ -98,14 +105,16 @@ def __init__(
             log("GH_TOKEN will be used to perform requests", first_line=True)
 
     def get_latest_release_version(self) -> str:
-        log(f"Get LATEST driver version for {self.browser_version}")
+        self.browser_version = firefox_version()
+        log(f"Get LATEST {self._name} version for {self.browser_version} firefox")
         resp = requests.get(
             url=self.latest_release_url,
             headers=self.auth_header,
             verify=self.ssl_verify,
         )
         validate_response(resp)
-        return resp.json()["tag_name"]
+        self._version = resp.json()["tag_name"]
+        return self._version
 
     def get_url(self):
         """Like https://github.com/mozilla/geckodriver/releases/download/v0.11.1/geckodriver-v0.11.1-linux64.tar.gz"""
@@ -185,7 +194,8 @@ def get_latest_release_version(self) -> str:
             for asset in release['assets']
             if asset['name'].startswith(self.get_name())
         )
-        return release['tag_name'].replace('selenium-', '')
+        self._version = release['tag_name'].replace('selenium-', '')
+        return self._version
 
     def get_url(self):
         """Like https://github.com/seleniumhq/selenium/releases/download/3.141.59/IEDriverServer_Win32_3.141.59.zip"""
@@ -249,7 +259,8 @@ def get_latest_release_version(self) -> str:
             verify=self.ssl_verify,
         )
         validate_response(resp)
-        return resp.json()["tag_name"]
+        self._version = resp.json()["tag_name"]
+        return self._version
 
     def get_url(self) -> str:
         # https://github.com/operasoftware/operachromiumdriver/releases/download/v.2.45/operadriver_linux64.zip
@@ -291,15 +302,22 @@ def __init__(
             url,
             latest_release_url,
         )
-        self.browser_version = get_browser_version_from_os(ChromeType.MSEDGE)
+        self.browser_version = ""
 
     def get_latest_release_version(self) -> str:
-        major_edge_version = self.browser_version.split(".")[0]
-        latest_release_url = {
-            OSType.WIN in self.get_os_type(): f'{self._latest_release_url}_{major_edge_version}_WINDOWS',
-            OSType.MAC in self.get_os_type(): f'{self._latest_release_url}_{major_edge_version}_MACOS',
-            OSType.LINUX in self.get_os_type(): f'{self._latest_release_url}_{major_edge_version}_LINUX',
-        }[True]
+        self.browser_version = get_browser_version_from_os(ChromeType.MSEDGE)
+        log(f"Get LATEST {self._name} version for {self.browser_version} Edge")
+        major_edge_version = self.browser_version.split(".")[0] if self.browser_version != 'UNKNOWN' else None
+        latest_release_url = (
+            {
+                OSType.WIN in self.get_os_type(): f'{self._latest_release_url}_{major_edge_version}_WINDOWS',
+                OSType.MAC in self.get_os_type(): f'{self._latest_release_url}_{major_edge_version}_MACOS',
+                OSType.LINUX in self.get_os_type(): f'{self._latest_release_url}_{major_edge_version}_LINUX',
+            }[True]
+            if self.browser_version != "UNKNOWN"
+            else self._latest_release_url.replace('LATEST_RELEASE', 'LATEST_STABLE')
+        )
         resp = requests.get(latest_release_url, verify=self.ssl_verify)
         validate_response(resp)
-        return resp.text.rstrip()
+        self._version = resp.text.rstrip()
+        return self._version

webdriver_manager/firefox.py

@@ -3,7 +3,6 @@
 from webdriver_manager import utils
 from webdriver_manager.driver import GeckoDriver
 from webdriver_manager.manager import DriverManager
-from webdriver_manager.logger import log
 
 
 class GeckoDriverManager(DriverManager):
@@ -27,5 +26,4 @@ def __init__(self, version="latest",
                                   mozila_release_tag=mozila_release_tag)
 
     def install(self):
-        log(f"Current firefox version is {self.driver.browser_version}", first_line=True)
         return self._get_driver_path(self.driver)

webdriver_manager/utils.py

@@ -79,7 +79,11 @@ def validate_response(resp):
     if resp.status_code == 404:
         raise ValueError("There is no such driver by url {}".format(resp.url))
     elif resp.status_code != 200:
-        raise ValueError(resp.json())
+        raise ValueError(
+            f'response body:\n{resp.json()}\n'
+            f'request url:\n{resp.request.url}\n'
+            f'response headers:\n{dict(resp.headers)}\n'
+        )
 
 
 def write_file(content, path):
@@ -149,35 +153,45 @@ def get_browser_version_from_os(browser_type=None):
     }
 
     cmd = cmd_mapping[browser_type][os_name()]
-    version = None
-    with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
-                          shell=True) as stream:
-        stdout = stream.communicate()[0].decode()
-        version = re.search(pattern, stdout)
+    version = read_version_from_cmd(cmd, pattern)
 
     if not version:
-        raise ValueError(
-            f'Could not get version for {browser_type} with the command: {cmd}'
-        )
-    current_version = version.group(0)
+        log(f'Could not get version for {browser_type} with the any command: {cmd}')
+
+    current_version = version.group(0) if version else 'UNKNOWN'
+
+    log(f"Current {browser_type} version is {current_version}")
     return current_version
 
 
 def firefox_version():
-    pattern = r'\d+.*'
+    pattern = r'(\d+.\d+)'
     cmd_mapping = {
         OSType.LINUX: 'firefox --version',
         OSType.MAC: r'/Applications/Firefox.app/Contents/MacOS/firefox --version',
         OSType.WIN: r"Powershell (Get-Item (Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\App Paths\firefox.exe').'(Default)').VersionInfo.ProductVersion",
     }
     cmd = cmd_mapping[os_name()]
-    version = None
-    with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
-                          shell=True) as stream:
-        stdout = stream.communicate()[0].decode()
-        version = re.search(pattern, stdout)
+
+    version = read_version_from_cmd(cmd, pattern)
 
     if not version:
-        raise ValueError(f'Could not get version for Firefox with this command: {cmd}')
-    current_version = version.group(0)
+        log(f'Could not get version for firefox with the any command: {cmd}')
+
+    current_version = version.group(0) if version else 'UNKNOWN'
+
+    log(f"Current firefox version is {current_version}")
     return current_version
+
+
+def read_version_from_cmd(cmd, pattern):
+    with subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            stdin=subprocess.DEVNULL,
+            shell=True,
+    ) as stream:
+        stdout = stream.communicate()[0].decode()
+        version = re.search(pattern, stdout)
+    return version