diff --git a/gnssanalysis/filenames.py b/gnssanalysis/filenames.py index 3789229..148f791 100644 --- a/gnssanalysis/filenames.py +++ b/gnssanalysis/filenames.py @@ -134,7 +134,7 @@ def determine_file_name_main( else: print(new_name) except NotImplementedError: - logging.warning(f"Skipping {f.name} as {f.suffix} files are not yet supported.") + warnings.warn(f"Skipping {f.name} as {f.suffix} files are not yet supported.") def determine_file_name( @@ -542,8 +542,8 @@ def determine_clk_name_props(file_path: pathlib.Path) -> dict[str, Any]: except Exception as e: # TODO: Work out what exceptions read_clk can actually throw when given a non-CLK file # At the moment we will also swallow errors we really shouldn't - logging.warning(f"{file_path.name} can't be read as an CLK file. Defaulting properties.") - logging.warning(f"Exception {e}, {type(e)}") + warnings.warn(f"{file_path.name} can't be read as an CLK file. Defaulting properties.") + warnings.warn(f"Exception {e}, {type(e)}") logging.info(traceback.format_exc()) return {} return name_props @@ -597,8 +597,8 @@ def determine_erp_name_props(file_path: pathlib.Path) -> dict[str, Any]: except Exception as e: # TODO: Work out what exceptions read_erp can actually throw when given a non-ERP file # At the moment we will also swallow errors we really shouldn't - logging.warning(f"{file_path.name} can't be read as an ERP file. Defaulting properties.") - logging.warning(f"Exception {e}, {type(e)}") + warnings.warn(f"{file_path.name} can't be read as an ERP file. Defaulting properties.") + warnings.warn(f"Exception {e}, {type(e)}") logging.info(traceback.format_exc()) return {} return name_props @@ -689,8 +689,8 @@ def determine_snx_name_props(file_path: pathlib.Path) -> dict[str, Any]: except Exception as e: # TODO: Work out what exceptions _get_snx_vector can actually throw when given a non-SNX file # At the moment we will also swallow errors we really shouldn't - logging.warning(f"{file_path.name} can't be read as an SNX file. Defaulting properties.") - logging.warning(f"Exception {e}, {type(e)}") + warnings.warn(f"{file_path.name} can't be read as an SNX file. Defaulting properties.") + warnings.warn(f"Exception {e}, {type(e)}") logging.info(traceback.format_exc()) return {} return name_props @@ -1015,7 +1015,7 @@ def check_filename_and_contents_consistency( # If parsing of a long filename fails, Project will not be present. In this case we have with minimal (and # maybe incorrect) properties to compare. So we raise a warning. if "project" not in file_name_properties: - logging.warning( + warnings.warn( f"Failed to parse filename according to the long filename format: '{input_file.name}'. " "As a result few useful properties are available to compare with the file contents, so the " "detailed consistency check will be skipped!" @@ -1027,7 +1027,7 @@ def check_filename_and_contents_consistency( contents_epoch_interval = file_content_properties.get("sampling_rate_seconds", None) if contents_epoch_interval is None: - logging.warning( + warnings.warn( f"Sampling rate couldn't be inferred from file contents '{input_file.name}'. " "Cannot allow for timespan discrepancies of one epoch interval, so an error may follow." ) @@ -1035,9 +1035,11 @@ def check_filename_and_contents_consistency( discrepancies = {} # Check for keys only present on one side orphan_keys = set(file_name_properties.keys()).symmetric_difference((set(file_content_properties.keys()))) - logging.warning( + orphan_keys_sorted = list(orphan_keys) + orphan_keys_sorted.sort() + warnings.warn( "The following properties can't be compared, as they were extracted only from file content or " - f"name (not both): {str(orphan_keys)}" + f"name (not both): {str(orphan_keys_sorted)}" ) if output_orphan_prop_names: # Output properties found only in content OR filename. diff --git a/gnssanalysis/gn_diffaux.py b/gnssanalysis/gn_diffaux.py index 3f29eea..4d0c6af 100644 --- a/gnssanalysis/gn_diffaux.py +++ b/gnssanalysis/gn_diffaux.py @@ -1,6 +1,7 @@ import logging as _logging from pathlib import Path as _Path from typing import Literal, Union +import warnings import numpy as _np import pandas as _pd @@ -118,7 +119,7 @@ def _compare_states(diffstd: _pd.DataFrame, log_lvl: int, tol: Union[float, None diff_states = diffstd.unstack(["TYPE", "SITE", "SAT", "BLK", "NUM"]) # we remove the '.droplevel("NUM", axis=0)' due to ORBIT_PTS non-uniqueness. Changing to ORBIT_PTS_blah might be a better solution if diff_states.empty: - _logging.warning(msg=f":diffutil states not present. Skipping") + warnings.warn(f":diffutil states not present. Skipping") return 0 if plot: # a standard scatter plot @@ -157,7 +158,7 @@ def _compare_residuals(diffstd: _pd.DataFrame, log_lvl: int, tol: Union[float, N idx_names_to_unstack.remove("TIME") # all but TIME: ['SITE', 'TYPE', 'SAT', 'NUM', 'It', 'BLK'] diff_residuals = diffstd.unstack(idx_names_to_unstack) if diff_residuals.empty: - _logging.warning(f":diffutil residuals not present. Skipping") + warnings.warn(f":diffutil residuals not present. Skipping") return 0 bad_residuals = _diff2msg(diff_residuals, tol=tol) if bad_residuals is not None: @@ -176,7 +177,7 @@ def _compare_residuals(diffstd: _pd.DataFrame, log_lvl: int, tol: Union[float, N def _compare_stec(diffstd, log_lvl, tol=None): stec_diff = diffstd.unstack(level=("SITE", "SAT", "LAYER")) if stec_diff.empty: - _logging.warning(f":diffutil stec states not present. Skipping") + warnings.warn(f":diffutil stec states not present. Skipping") return 0 bad_sv_states = _diff2msg(stec_diff, tol, dt_as_gpsweek=True) if bad_sv_states is not None: @@ -323,7 +324,7 @@ def compare_clk( clk_b: baseline (normally b is test) """ - _logging.warning( + warnings.warn( "compare_clk() is deprecated. Please use diff_clk() and note that the clk inputs are in opposite order" ) return diff_clk(clk_baseline=clk_b, clk_test=clk_a, norm_types=norm_types, ext_dt=ext_dt, ext_svs=ext_svs) @@ -428,7 +429,7 @@ def sisre( DEPRECATED """ - _logging.warning( + warnings.warn( "sisre() is deprecated, please use calculate_sisre() instead. Note that input arg test/baseline orders are flipped" ) return calculate_sisre( diff --git a/gnssanalysis/gn_download.py b/gnssanalysis/gn_download.py index 767897d..7ddf9a1 100644 --- a/gnssanalysis/gn_download.py +++ b/gnssanalysis/gn_download.py @@ -94,12 +94,11 @@ def __call__(self, bytes_transferred): _sys.stdout.flush() - -def get_earthdata_credentials(username: str = None, password: str = None) -> Tuple[str, str]: +def get_earthdata_credentials(username: Optional[str] = None, password: Optional[str] = None) -> Tuple[str, str]: """ Get NASA Earthdata credentials from .netrc file or direct parameters. - :param str username: Directly provided username (highest priority) - :param str password: Directly provided password (highest priority) + :param Optional[str] username: Directly provided username (highest priority) + :param Optional[str] password: Directly provided password (highest priority) :return Tuple[str, str]: Username and password tuple :raises ValueError: If no credentials can be obtained """ @@ -774,17 +773,17 @@ def ftp_tls(url: str, **kwargs) -> Generator[Any, Any, Any]: def download_file_from_cddis( filename: str, - ftp_folder: Optional[str] = None, # deprecated - url_folder: Optional[str] = None, # preferred + ftp_folder: Optional[str] = None, # deprecated + url_folder: Optional[str] = None, # preferred output_folder: _Path = _Path("."), max_retries: int = 3, decompress: bool = True, if_file_present: str = "prompt_user", - username: str = None, - password: str = None, + username: Optional[str] = None, + password: Optional[str] = None, note_filetype: Optional[str] = None, ) -> Union[_Path, None]: - """ Download a single file from the CDDIS HTTPS archive using NASA Earthdata authentication + """Download a single file from the CDDIS HTTPS archive using NASA Earthdata authentication :param str filename: Name of the file to download :param str ftp_folder: (Deprecated) Legacy folder path on the CDDIS FTP server. Use url_folder instead @@ -794,8 +793,8 @@ def download_file_from_cddis( :param bool decompress: If true, decompresses files on download, defaults to True :param str if_file_present: What to do if file already present: "replace", "dont_replace", defaults to "prompt_user" :param str note_filetype: How to label the file for STDOUT messages, defaults to None - :param str username: NASA Earthdata username (optional, will try .netrc if not provided). - :param str password: NASA Earthdata password (optional, will try .netrc if not provided). + :param Optional[str] username: NASA Earthdata username (optional, will try .netrc if not provided). + :param Optional[str] password: NASA Earthdata password (optional, will try .netrc if not provided). :raises ValueError: If no credentials can be obtained. :raises requests.RequestException: If the file cannot be downloaded after retries. :return _Path or None: The pathlib.Path of the downloaded file (or decompressed output of it). @@ -823,14 +822,8 @@ def download_file_from_cddis( if download_filepath is None: return None # File exists and user chose not to replace - # Get NASA Earthdata credentials - try: - earthdata_username, earthdata_password = get_earthdata_credentials( - username=username, password=password - ) - except ValueError as e: - logging.error(f"Failed to obtain NASA Earthdata credentials: {e}") - raise + # Get NASA Earthdata credentials (raises ValueError on failure) + earthdata_username, earthdata_password = get_earthdata_credentials(username=username, password=password) retries = 0 while retries <= max_retries: @@ -862,13 +855,15 @@ def download_file_from_cddis( except _requests.exceptions.RequestException as e: retries += 1 if retries > max_retries: + # TODO consider wrapping the RequestException with this, and raising that, rather than logging an error logging.error(f"Failed to download {filename} after {max_retries} retries: {e}") if download_filepath.is_file(): download_filepath.unlink() raise backoff = _random.uniform(0.0, 2.0 ** retries) - logging.warning(f"Error downloading {filename}: {e} " - f"(retry {retries}/{max_retries}, backoff {backoff:.1f}s)") + _warnings.warn( + f"Error downloading {filename}: {e} " f"(retry {retries}/{max_retries}, backoff {backoff:.1f}s)" + ) _time.sleep(backoff) raise Exception("Unexpected fallthrough in download_file_from_cddis.") @@ -876,11 +871,11 @@ def download_file_from_cddis( def download_multiple_files_from_cddis( files: List[str], - ftp_folder: Optional[str] = None, # deprecated - url_folder: Optional[str] = None, # preferred + ftp_folder: Optional[str] = None, # deprecated + url_folder: Optional[str] = None, # preferred output_folder: _Path = _Path("."), - username: str = None, - password: str = None, + username: Optional[str] = None, + password: Optional[str] = None, ) -> None: """ Download multiple files from the CDDIS HTTPS archive concurrently, using a thread pool. @@ -889,8 +884,8 @@ def download_multiple_files_from_cddis( :param str ftp_folder: (Deprecated) Legacy folder path on the CDDIS FTP server. Use url_folder instead. :param str url_folder: Folder path (relative to CDDIS HTTPS archive root). :param _Path output_folder: Local folder to store the output files. - :param str username: NASA Earthdata username (optional, will try .netrc if not provided). - :param str password: NASA Earthdata password (optional, will try .netrc if not provided). + :param Optional[str] username: NASA Earthdata username (optional, will try .netrc if not provided). + :param Optional[str] password: NASA Earthdata password (optional, will try .netrc if not provided). :raises ValueError: If both ftp_folder and url_folder are provided. :return None: Runs downloads in parallel, does not return values. Each file is handled independently. """ @@ -943,8 +938,8 @@ def download_product_from_cddis( project_type: str = "OPS", timespan: _datetime.timedelta = _datetime.timedelta(days=2), if_file_present: str = "prompt_user", - username: str = None, - password: str = None, + username: Optional[str] = None, + password: Optional[str] = None, ) -> List[_Path]: """Download the file/s from CDDIS based on start and end epoch, to the download directory (download_dir) @@ -961,8 +956,8 @@ def download_product_from_cddis( :param str project_type: Project type of file to download (e.g. ), defaults to "OPS" :param _datetime.timedelta timespan: Timespan of the file/s to download, defaults to _datetime.timedelta(days=2) :param str if_file_present: What to do if file already present: "replace", "dont_replace", defaults to "prompt_user" - :param str username: NASA Earthdata username (optional, will try .netrc if not provided). - :param str password: NASA Earthdata password (optional, will try .netrc if not provided). + :param Optional[str] username: NASA Earthdata username (optional, will try .netrc if not provided). + :param Optional[str] password: NASA Earthdata password (optional, will try .netrc if not provided). :raises FileNotFoundError: Raise error if the specified file cannot be found on CDDIS :raises Exception: If a file fails to download after all retries. :return List[_Path]: List of pathlib.Path objects to downloaded (or decompressed) files. diff --git a/gnssanalysis/gn_io/sp3.py b/gnssanalysis/gn_io/sp3.py index 7766f1c..2938e59 100644 --- a/gnssanalysis/gn_io/sp3.py +++ b/gnssanalysis/gn_io/sp3.py @@ -368,7 +368,7 @@ def update_sp3_comments( if not validate_sp3_comment_lines( new_lines, strict_mode=strict_mode, attempt_fixes=True, fail_on_fixed_issues=False ): - logger.warning("SP3 comment validation identified unfixable issues while writing comments!") + warnings.warn("SP3 comment validation identified unfixable issues while writing comments!") # Write updated lines back to the DataFrame attributes sp3_df.attrs["COMMENTS"] = new_lines @@ -537,9 +537,7 @@ def _check_column_alignment_of_sp3_block( f"Epoch header should be {_SP3_EPOCH_HEADER_WIDTH} chars long, but was {len(date)}: '{date}'" ) elif strict_mode == StrictModes.STRICT_WARN: - logger.warning( - f"Epoch header should be {_SP3_EPOCH_HEADER_WIDTH} chars long, but was {len(date)}: '{date}'" - ) + warnings.warn(f"Epoch header should be {_SP3_EPOCH_HEADER_WIDTH} chars long, but was {len(date)}: '{date}'") epoch_header_offset = 0 # TODO remove this after fixing our block splitting logic to not remove the '*' if len(date) == _SP3_EPOCH_HEADER_WIDTH - 1: # Cut short by our block splitting logic. Adjust indexes accordingly. epoch_header_offset = -1 @@ -549,7 +547,7 @@ def _check_column_alignment_of_sp3_block( if strict_mode == StrictModes.STRICT_RAISE: raise ValueError(f"Misaligned epoch header line (unused column didn't contain a space): '{date}'") elif strict_mode == StrictModes.STRICT_WARN: - logger.warning(f"Misaligned epoch header line (unused column didn't contain a space): '{date}'") + warnings.warn(f"Misaligned epoch header line (unused column didn't contain a space): '{date}'") # Now check each data line for this epoch @@ -568,7 +566,7 @@ def _check_column_alignment_of_sp3_block( f"Data lines should be {_SP3_DATA_LINE_WIDTH} chars. Got one {line_length} chars long: '{line}'" ) elif strict_mode == StrictModes.STRICT_WARN: - logger.warning( + warnings.warn( f"Data lines should be {_SP3_DATA_LINE_WIDTH} chars. Got one {line_length} chars long: '{line}'" ) @@ -587,7 +585,7 @@ def _check_column_alignment_of_sp3_block( if strict_mode == StrictModes.STRICT_RAISE: raise ValueError(f"Data line should start with P/V/EP/EV. First two chars were: '{line[:2]}'") elif strict_mode == StrictModes.STRICT_WARN: - logger.warning(f"Data line should start with P/V/EP/EV. First two chars were: '{line[:2]}'") + warnings.warn(f"Data line should start with P/V/EP/EV. First two chars were: '{line[:2]}'") # Can't check column alignment for this line as we don't know which record type it is. continue @@ -600,7 +598,7 @@ def _check_column_alignment_of_sp3_block( if strict_mode == StrictModes.STRICT_RAISE: raise ValueError(f"Misaligned data line (unused column did not contain a space): '{line}'") elif strict_mode == StrictModes.STRICT_WARN: - logger.warning(f"Misaligned data line (unused column did not contain a space): '{line}'") + warnings.warn(f"Misaligned data line (unused column did not contain a space): '{line}'") def _process_sp3_block( @@ -674,7 +672,7 @@ def try_get_sp3_filename(path_or_bytes: Union[str, Path, bytes]) -> Union[str, N if isinstance(path_or_bytes, str): return path_or_bytes.rsplit("/")[-1] - logger.warning("sp3_path_or_bytes was of an unexpected type. Filename not extracted") + warnings.warn("sp3_path_or_bytes was of an unexpected type. Filename not extracted") return None @@ -716,7 +714,7 @@ def check_epoch_counts_for_discrepancies( f"{content_unique_epoch_count} (unique) epochs in the content (duplicate epoch check comes later)." ) elif strict_mode == StrictModes.STRICT_WARN: - logger.warning( + warnings.warn( f"Header says there should be {header_epoch_count} epochs, however there are " f"{content_unique_epoch_count} (unique) epochs in the content (duplicate epoch check comes later)." ) @@ -736,7 +734,7 @@ def check_epoch_counts_for_discrepancies( if strict_mode == StrictModes.STRICT_RAISE: raise ValueError(f"Failed to get timespan from filename '{sp3_filename}'") elif strict_mode == StrictModes.STRICT_WARN: - logger.warning(f"Failed to get timespan from filename '{sp3_filename}'") + warnings.warn(f"Failed to get timespan from filename '{sp3_filename}'") return filename_sample_rate = filename_props.get("sampling_rate") @@ -744,7 +742,7 @@ def check_epoch_counts_for_discrepancies( if strict_mode == StrictModes.STRICT_RAISE: raise ValueError(f"Failed to get sampling_rate from filename '{sp3_filename}'") elif strict_mode == StrictModes.STRICT_WARN: - logger.warning(f"Failed to get sampling_rate from filename '{sp3_filename}'") + warnings.warn(f"Failed to get sampling_rate from filename '{sp3_filename}'") return filename_sample_rate_timedelta = filenames.convert_nominal_span(filename_sample_rate) @@ -765,7 +763,7 @@ def check_epoch_counts_for_discrepancies( f"there should be {filename_derived_epoch_count} (or {filename_derived_epoch_count-1} at minimum)." ) elif strict_mode == StrictModes.STRICT_WARN: - logger.warning( + warnings.warn( f"Header says there should be {header_epoch_count} epochs, however filename '{sp3_filename}' implies " f"there should be {filename_derived_epoch_count} (or {filename_derived_epoch_count-1} at minimum)." ) @@ -802,7 +800,7 @@ def check_sp3_version(sp3_bytes: bytes, strict_mode: type[StrictMode] = StrictMo f"Support for SP3 file version '{version_char_as_string}' is untested. Refusing to read as strict mode is on." ) elif strict_mode == StrictModes.STRICT_WARN: - logger.warning( + warnings.warn( f"Reading an older SP3 file version '{version_char_as_string}'. This may not parse correctly!" ) return False @@ -871,7 +869,7 @@ def validate_sp3_comment_lines( f"SP3 files must have at least 4 comment lines! File is {short_by_lines} short of that" ) elif strict_mode == StrictModes.STRICT_WARN: - logger.warning(f"SP3 files must have at least 4 comment lines! File is {short_by_lines} short of that") + warnings.warn(f"SP3 files must have at least 4 comment lines! File is {short_by_lines} short of that") if attempt_fixes: sp3_comment_lines.extend([SP3_COMMENT_START] * short_by_lines) @@ -888,7 +886,7 @@ def validate_sp3_comment_lines( if strict_mode == StrictModes.STRICT_RAISE: raise ValueError(f"SP3 comments must begin with '/* ' (note space). Line: '{sp3_comment_lines[i]}'") elif strict_mode == StrictModes.STRICT_WARN: - logger.warning(f"SP3 comments must begin with '/* ' (note space). Line: '{sp3_comment_lines[i]}'") + warnings.warn(f"SP3 comments must begin with '/* ' (note space). Line: '{sp3_comment_lines[i]}'") if attempt_fixes: if sp3_comment_lines[i][0:2] == "/*": @@ -909,7 +907,7 @@ def validate_sp3_comment_lines( f"Line (length {len(sp3_comment_lines[i])}): '{sp3_comment_lines[i]}'" ) elif strict_mode == StrictModes.STRICT_WARN: - logger.warning( + warnings.warn( "SP3 comment lines must not exceed 80 chars (including lead-in). " f"Line (length {len(sp3_comment_lines[i])}): '{sp3_comment_lines[i]}'" ) @@ -1052,7 +1050,7 @@ def read_sp3( if strict_mode == StrictModes.STRICT_RAISE: raise NotImplementedError("EP and EV flag rows are currently not supported") elif strict_mode == StrictModes.STRICT_WARN: - logger.warning( + warnings.warn( "EP / EV flag rows encountered. These are not yet supported. Dropping them from DataFrame. " "Switch to strict mode RAISE to raise an exception instead" ) @@ -1131,7 +1129,7 @@ def read_sp3( f"Number of SVs in SP3 header ({header_sv_count}) did not match file contents ({content_sv_count})!" ) if discrepancy_strictness == StrictModes.STRICT_WARN: - logger.warning( + warnings.warn( f"Number of SVs in SP3 header ({header_sv_count}) did not match file contents ({content_sv_count})!" ) @@ -1160,7 +1158,7 @@ def read_sp3( f"SP3 path is: '{description_for_path_or_bytes(sp3_path_or_bytes)}'." ) elif strictness_dupes == StrictModes.STRICT_WARN: - logger.warning( + warnings.warn( f"Duplicate epoch(s) found in SP3 ({duplicated_indexes.sum()} additional entries, potentially non-unique). " f"First duplicate (as J2000): {first_dupe} (as date): {first_dupe + _gn_const.J2000_ORIGIN} " f"SP3 path is: '{description_for_path_or_bytes(sp3_path_or_bytes)}'. Duplicates will be removed, keeping first." @@ -1236,7 +1234,7 @@ def parse_sp3_header(header: bytes, warn_on_negative_sv_acc_values: bool = True) if len(sv_regex_matches) != 0: # Result found head_sv_expected_count = int(sv_regex_matches[0][0]) # Line 1, group 1 else: - logger.warning("Failed to extract count of expected SVs from SP3 header.") + warnings.warn("Failed to extract count of expected SVs from SP3 header.") # Get second capture group from each match, concat into byte string. These are the actual SVs. i.e. 'G02G03G04'... sv_id_matches = b"".join([x[1] for x in sv_regex_matches]) @@ -1246,7 +1244,7 @@ def parse_sp3_header(header: bytes, warn_on_negative_sv_acc_values: bool = True) # Sanity check that the number of SVs the regex found, matches what the header said should be there. found_sv_count = head_svs.shape[0] # Effectively len() of the SVs array. Note this could include null/NA/NaN if head_sv_expected_count is not None and found_sv_count != head_sv_expected_count: - logger.warning( + warnings.warn( "Number of Satellite Vehicle (SV) entries extracted from the SP3 header, did not match the " "number of SVs the header said were there! This might be a header writer or header parser bug! " f"SVs extracted: {found_sv_count}, SV count given by header: {head_sv_expected_count} " @@ -1269,7 +1267,7 @@ def parse_sp3_header(header: bytes, warn_on_negative_sv_acc_values: bool = True) sv_tbl = _pd.Series(head_svs_std, index=head_svs) if warn_on_negative_sv_acc_values and any(acc < 0 for acc in head_svs_std): - logger.warning( + warnings.warn( "SP3 header contained orbit accuracy codes which were negative! These values represent " "error expressed as 2^x mm, so negative values are unrealistic and likely an error. " f"Parsed SVs and ACCs: {sv_tbl}" @@ -1306,11 +1304,11 @@ def parse_sp3_header(header: bytes, warn_on_negative_sv_acc_values: bool = True) header_version = str(header_array[0]) if header_version in ("a", "b"): - logger.warning(f"SP3 file is old version: '{header_version}', you may experience parsing issues") + warnings.warn(f"SP3 file is old version: '{header_version}', you may experience parsing issues") elif header_version in ("c", "d"): logger.info(f"SP3 header states SP3 file version is: {header_array[0]}") else: - logger.warning( + warnings.warn( f"SP3 header is of an unknown version, or failed to parse! Version appears to be: '{header_version}'" ) @@ -1335,7 +1333,7 @@ def clean_sp3_orb(sp3_df: _pd.DataFrame, use_offline_sat_removal: bool) -> _pd.D try: filename = _os.path.basename(sp3_df.attrs["path"]) except Exception as e: - logger.error(f"Failed to grab filename from sp3 dataframe for error output purposes: {str(e)}") + warnings.warn(f"Failed to grab filename from sp3 dataframe for error output purposes: {str(e)}") if sp3_df.size == 0: raise ValueError(f"Bad input data: can't clean an empty SP3 DataFrame. Source filename: '{filename}'") @@ -1440,7 +1438,7 @@ def get_unique_svs(sp3_df: _pd.DataFrame) -> _pd.Index: # -> In this case the PV_FLAG index level will have been dropped. if "PV_FLAG" in sp3_df.index.names: if "E" in sp3_df.index.get_level_values("PV_FLAG").unique(): - logger.warning( + warnings.warn( "EV/EP record found late in SP3 processing. Until we actually support them, " "they should be removed by the EV/EP check earlier on! Filtering out while determining unique SVs." ) @@ -1492,7 +1490,7 @@ def gen_sp3_header( sv_tbl = header.SV_INFO if head.VERSION != "d": - logger.warning( + warnings.warn( f"Stored SP3 header indicates version '{head.VERSION}'. Changing to version 'd' for " "write-out given that's the version this implementation is designed to create" ) @@ -1601,7 +1599,7 @@ def gen_sp3_header( attempt_fixes=False, fail_on_fixed_issues=True, ): - logger.warning("SP3 comments failed validation while being read in. Please see above logs for details.") + warnings.warn("SP3 comments failed validation while being read in. Please see above logs for details.") # Put the newlines back on the end of each comment line, before merging into the output header sp3_comment_lines = [line + "\n" for line in sp3_comment_lines] @@ -1659,7 +1657,7 @@ def gen_sp3_content( raise NotImplementedError("Output of SP3 velocity data not currently supported") # Drop any of the defined velocity columns that are present, so it doesn't mess up the output. - logger.warning("SP3 velocity output not currently supported! Dropping velocity columns before writing out.") + warnings.warn("SP3 velocity output not currently supported! Dropping velocity columns before writing out.") # Remove any defined velocity column we have, don't raise exceptions for defined vel columns we may not have: out_df = out_df.drop(columns=SP3_VELOCITY_COLUMNS[1], errors="ignore") @@ -2082,7 +2080,7 @@ def diff_sp3_rac( sp3_test = clean_sp3_orb(sp3_test, use_offline_sat_removal) if use_cubic_spline and not use_offline_sat_removal: - logger.warning( + warnings.warn( "Caution: use_cubic_spline is enabled, but use_offline_sat_removal is not. If there are any nodata " "position values, the cubic interpolator will crash!" ) diff --git a/requirements.txt b/requirements.txt index 1a6a1c6..7d4b3ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ hatanaka jinja2 matplotlib numpy -pandas +pandas==2.3.3 # Temporarily pinned in Jan 2026, while fixing our compatibility issues with Pandas 3 plotext==4.2 plotly pyfakefs diff --git a/tests/test_clk.py b/tests/test_clk.py index 58dcc5a..5e1fb05 100644 --- a/tests/test_clk.py +++ b/tests/test_clk.py @@ -47,15 +47,29 @@ def test_diff_clk(self): clk_df_gfz = clk.read_clk(clk_path=file_paths[1]) # Deprecated version - result_default = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz) - result_daily_only = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["daily"]) - result_epoch_only = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["epoch"]) - result_sv_only = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["sv"]) # G01 ref - result_G06 = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["G06"]) - result_daily_epoch_G04 = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["daily", "epoch", "G04"]) - result_epoch_G07 = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["epoch", "G07"]) - result_daily_G08 = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["daily", "G08"]) - result_G09_G11 = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["G09", "G11"]) + # Ensure depreciation warnings are raised, but don't print them. + with self.assertWarns(Warning) as warning_assessor: + result_default = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz) + result_daily_only = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["daily"]) + result_epoch_only = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["epoch"]) + result_sv_only = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["sv"]) # G01 ref + result_G06 = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["G06"]) + result_daily_epoch_G04 = gn_diffaux.compare_clk( + clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["daily", "epoch", "G04"] + ) + result_epoch_G07 = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["epoch", "G07"]) + result_daily_G08 = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["daily", "G08"]) + result_G09_G11 = gn_diffaux.compare_clk(clk_a=clk_df_igs, clk_b=clk_df_gfz, norm_types=["G09", "G11"]) + captured_warnings = warning_assessor.warnings + self.assertEqual( + "compare_clk() is deprecated. Please use diff_clk() and note that the clk inputs are in opposite order", + str(captured_warnings[0].message), + ) + self.assertEqual( + len(captured_warnings), + 9, + "Expected exactly 9 warnings. Check what other warnings are being raised!", + ) # Test index is as expected self.assertEqual(result_default.index[0], 760708800) diff --git a/tests/test_datasets/sp3_incorrect_timerange.py b/tests/test_datasets/sp3_incorrect_timerange.py index e0fe13d..41ade9f 100644 --- a/tests/test_datasets/sp3_incorrect_timerange.py +++ b/tests/test_datasets/sp3_incorrect_timerange.py @@ -4,6 +4,7 @@ # speed. NOTE: at the time of writing the main bottleneck in reading seems to be the fact each epoch is parsed into a # temporary DataFrame, so without significantly overhauling that code, number of epochs appears to be the main # determiner of speed. +# NOTE: Edited to have technically correct SP3 comment lines (trailing space on '/* '). sp3_test_inconsistent_timerange = b"""#dP2024 1 27 0 0 0.0000000 289 ORBIT IGS14 FIT GAA ## 2298 518400.00000000 300.00000000 60336 0.0000000000000 + 1 G02 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 @@ -22,10 +23,10 @@ %f 0.0000000 0.000000000 0.00000000000 0.000000000000000 %i 0 0 0 0 0 0 0 0 0 %i 0 0 0 0 0 0 0 0 0 -/* -/* +/* +/* /* ---- FILE NOT FOR OPERATIONAL USE ---- EXPERIMENTAL ---- -/* +/* * 2024 1 27 0 0 0.00000000 PG02 20022.878477 10575.933444 14581.906269 -494.899674 * 2024 1 27 0 5 0.00000000 diff --git a/tests/test_datasets/sp3_test_data.py b/tests/test_datasets/sp3_test_data.py index 71cd6d7..ce3c84b 100644 --- a/tests/test_datasets/sp3_test_data.py +++ b/tests/test_datasets/sp3_test_data.py @@ -18,7 +18,8 @@ """ # first dataset is part of the IGS benchmark (modified to include non null data on clock) -sp3_test_data_igs_benchmark_null_clock = b"""#dV2007 4 12 0 0 0.00000000 2 ORBIT IGS14 BHN ESOC +# Header adjusted to 3 epochs, reflecting content. +sp3_test_data_igs_benchmark_null_clock = b"""#dV2007 4 12 0 0 0.00000000 3 ORBIT IGS14 BHN ESOC ## 1422 345600.00000000 900.00000000 54202 0.0000000000000 + 2 G01G02 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 diff --git a/tests/test_download.py b/tests/test_download.py index 118bc08..a58591a 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -238,7 +238,6 @@ def setUp(self): """Set up fake filesystem for each test.""" self.setUpPyfakefs() - @patch('gnssanalysis.gn_download.get_earthdata_credentials') @patch('gnssanalysis.gn_download.check_whether_to_download') def test_file_already_exists_skip(self, mock_check, mock_creds): @@ -361,12 +360,23 @@ def test_retry_logic_with_eventual_success(self, mock_sleep, mock_session, mock_ mock_session_instance.get.side_effect = [mock_response_fail, mock_response_success] # Execute - result = download_file_from_cddis( - filename="test.txt", - url_folder="test/folder", - output_folder=output_dir, - max_retries=2, - decompress=False + # Check (and don't print) warning + with self.assertWarns(Warning) as warning_assessor: + result = download_file_from_cddis( + filename="test.txt", url_folder="test/folder", output_folder=output_dir, max_retries=2, decompress=False + ) + + captured_warnings = warning_assessor.warnings + + self.assertIn( # Backoff time is random, so can't be explicitly matched here. + "Error downloading test.txt: Network error (retry 1/2, backoff", + str(captured_warnings[0].message), + ) + + self.assertEqual( + len(captured_warnings), + 1, + "Expected 1 warning. Check what other warnings are being raised!", ) # Verify @@ -374,7 +384,6 @@ def test_retry_logic_with_eventual_success(self, mock_sleep, mock_session, mock_ self.assertEqual(mock_session_instance.get.call_count, 2) mock_sleep.assert_called_once() # Should sleep once between retries - @patch('gnssanalysis.gn_download.get_earthdata_credentials') @patch('gnssanalysis.gn_download.check_whether_to_download') @patch('requests.Session') diff --git a/tests/test_filenames.py b/tests/test_filenames.py index b7c40b9..e129b98 100644 --- a/tests/test_filenames.py +++ b/tests/test_filenames.py @@ -42,17 +42,42 @@ def test_determine_properties_from_contents(self): sp3_compliant_filename = Path(path_string_compliant) # Run - # TODO we don't test for this warning apart from with SP3 for now. - with self.assertWarns(Warning): - # Temporary, until we confirm warnings are appearing in standard logs. Then logging.warning() call can go. - logging.disable(logging.WARNING) + # TODO For now, we only test this with SP3 files. + with self.assertWarns(Warning) as warning_assessor: + # NOTE: this only meaningfully tests determine_sp3_name_props(), and only really the filename # (not content) based parts of this: + + # Should raise a warning about the non-compliant filename: derived_from_noncompliant = filenames.determine_properties_from_contents_and_filename( sp3_noncompliant_filename ) - logging.disable(logging.NOTSET) - derived_from_compliant = filenames.determine_properties_from_contents_and_filename(sp3_compliant_filename) + + # Should raise a warning about the epoch count mismatch (filename otherwise valid) + derived_from_compliant = filenames.determine_properties_from_contents_and_filename(sp3_compliant_filename) + + captured_warnings = warning_assessor.warnings + self.assertIn( + "Filename failed overly permissive regex for IGS short format", + str(captured_warnings[0].message), + ) + self.assertEqual( + "Failed to get timespan from filename 'file1.sp3'", + str(captured_warnings[1].message), + ) + + # TODO warning 3 (index 2), is a duplicate of the first warning. Check the stack to see if this makes sense for + # the call order. + + self.assertEqual( + "Header says there should be 2 epochs, however filename 'COD0OPSFIN_20242010000_01D_05M_ORB.SP3' implies there should be 288 (or 287 at minimum).", + str(captured_warnings[-1].message), + ) + self.assertEqual( + len(captured_warnings), + 4, + "Expected 4 warnings. Check what other warnings are being raised!", + ) # Verify # These are computed values at time of wrting: @@ -197,14 +222,48 @@ def test_determine_file_name(self): sp3_noncompliant_filename = Path(fake_path_noncompliant) sp3_compliant_filename = Path(fake_path_compliant) - # Require a warning. Also silences warning (would normally be routed to logging) while running the test. - with self.assertWarns(Warning): - # Temporary, until we confirm warnings are appearing in standard logs. Then logging.warning() call can go. - logging.disable(logging.WARNING) + # Require warnings. Also silences warnings (would normally be routed to logging) while running the test. + with self.assertWarns(Warning) as warning_assessor: + derived_filename_noncompliant_input = filenames.determine_file_name(sp3_noncompliant_filename) - logging.disable(logging.NOTSET) + # Expect + # - 'Filename failed overly permissive regex for IGS short format': 'file2.sp3'. Will attempt to parse, but output will likely be wrong' + # - 'Failed to get timespan from filename 'file2.sp3'' + + captured_warnings = warning_assessor.warnings + self.assertEqual( + "Filename failed overly permissive regex for IGS short format': 'file2.sp3'. Will attempt to parse, but output will likely be wrong", + str(captured_warnings[0].message), + ) + self.assertEqual( + "Failed to get timespan from filename 'file2.sp3'", + str(captured_warnings[1].message), + ) - derived_filename_compliant_input = filenames.determine_file_name(sp3_compliant_filename) + # TODO warning 3 (index 2), is a duplicate of the first warning. Check the stack to see if this makes sense for + # the call order. + + self.assertEqual( + len(captured_warnings), + 3, + "Expected 3 warnings. Check what other warnings are being raised!", + ) + + with self.assertWarns(Warning) as warning_assessor: + derived_filename_compliant_input = filenames.determine_file_name(sp3_compliant_filename) + # Expect: + # - 'Header says there should be 2 epochs, however filename 'COD0OPSFIN_20242010000_01D_05M_ORB.sp3' implies there should be 288 (or 287 at minimum).' + + captured_warnings = warning_assessor.warnings + self.assertEqual( + "Header says there should be 2 epochs, however filename 'COD0OPSFIN_20242010000_01D_05M_ORB.sp3' implies there should be 288 (or 287 at minimum).", + str(captured_warnings[0].message), + ) + self.assertEqual( + len(captured_warnings), + 1, + "Expected 1 warning. Check what other warnings are being raised!", + ) expected_filename_noncompliant_input = "FIL0EXP_20242010000_05M_05M_ORB.SP3" expected_filename_compliant_input = "COD0OPSFIN_20242010000_05M_05M_ORB.SP3" @@ -221,7 +280,33 @@ def test_check_discrepancies(self): self.fs.create_file(fake_path_string, contents=sp3_test_inconsistent_timerange) test_sp3_file = Path(fake_path_string) - discrepant_properties = filenames.check_filename_and_contents_consistency(test_sp3_file) + # Check warnings, prevent printing + with self.assertWarns(Warning) as warning_assessor: + + discrepant_properties = filenames.check_filename_and_contents_consistency(test_sp3_file) + # - Expect epoch mismatch warning: the very thing this check is designed to detect. + # - Expect missing key 'sampling_rate_seconds' from filename. This key can be present in content properties, + # and preserves the parsed seconds before conversion to a span string (e.g. 05M). + + captured_warnings = warning_assessor.warnings + + # Expect epoch mismatch warning: the very thing this check is designed to detect. + self.assertEqual( + "Header says there should be 289 epochs, however filename 'GAG0EXPULT_20240270000_02D_05M_ORB.SP3' implies there should be 576 (or 575 at minimum).", + str(captured_warnings[0].message), + ) + + self.assertEqual( + "The following properties can't be compared, as they were extracted only from file content or name (not both): ['end_epoch', 'sampling_rate_seconds']", + str(captured_warnings[1].message), + ) + + self.assertEqual( + len(captured_warnings), + 2, + "Expected 2 warnings. Check what other warnings are being raised!", + ) + expected_discrepant_properties = {"timespan": (timedelta(days=2), timedelta(days=1))} self.assertEqual(discrepant_properties, expected_discrepant_properties) diff --git a/tests/test_sp3.py b/tests/test_sp3.py index a381041..15cd553 100644 --- a/tests/test_sp3.py +++ b/tests/test_sp3.py @@ -1,6 +1,5 @@ from datetime import timedelta import unittest -from unittest.mock import patch, mock_open from pyfakefs.fake_filesystem_unittest import TestCase import numpy as np @@ -73,16 +72,35 @@ def test_check_sp3_version(self): sp3.check_sp3_version(fake_header_version_e) # Ambiguous cases + # Silence and check warnings about SP3 version B and C potential incompatibility. + with self.assertWarns(Warning) as warning_assessor: + + self.assertEqual( + sp3.check_sp3_version(fake_header_version_b), + False, + "SP3 version b should not be considered fully supported", + ) + + self.assertEqual( + sp3.check_sp3_version(fake_header_version_c), + False, + "SP3 version c should not be considered fully supported", + ) + + # Assess the warnings raised by those two checks + captured_warnings = warning_assessor.warnings self.assertEqual( - sp3.check_sp3_version(fake_header_version_b), - False, - "SP3 version b should not be considered fully supported", + "Reading an older SP3 file version 'b'. This may not parse correctly!", str(captured_warnings[0].message) ) self.assertEqual( - sp3.check_sp3_version(fake_header_version_c), - False, - "SP3 version c should not be considered fully supported", + "Reading an older SP3 file version 'c'. This may not parse correctly!", str(captured_warnings[1].message) ) + self.assertEqual( + len(captured_warnings), + 2, + "Expected only 2 warnings. Check what other warnings are being raised!", + ) + # Our best supported version should return True self.assertEqual( sp3.check_sp3_version(fake_header_version_d), True, "SP3 version d should be considered best supported" @@ -206,14 +224,20 @@ def test_read_sp3_overlong_lines(self): # """ # sp3.read_sp3(test_content_no_overlong) - with self.assertRaises(ValueError) as read_exception: - sp3.read_sp3(test_content_no_overlong, strictness_comments=STRICT_OFF, strict_mode=STRICT_RAISE) - self.assertEqual( - str(read_exception.exception), "2 SP3 epoch data lines were overlong and very likely to parse incorrectly." - ) + with self.assertWarns(Warning) as warning_assessor: + + with self.assertRaises(ValueError) as read_exception: + sp3.read_sp3(test_content_no_overlong, strictness_comments=STRICT_OFF, strict_mode=STRICT_RAISE) + self.assertEqual( + str(read_exception.exception), + "2 SP3 epoch data lines were overlong and very likely to parse incorrectly.", + ) + captured_warnings = warning_assessor.warnings + self.assertIn("Line of SP3 input exceeded max width:", str(captured_warnings[0].message)) # # Assert that it still warns by default (NOTE: we can't test this with above example data, as it doens't # # contain a full header) + # TODO # with self.assertWarns(Warning) as read_warning: # sp3.read_sp3(test_content_no_overlong, strictness_comments=STRICT_OFF) # self.assertEqual( @@ -426,7 +450,19 @@ def test_clean_sp3_orb(self): ) # Test cleaning function without offline sat removal - sp3_df_no_offline_removal = sp3.clean_sp3_orb(sp3_df, False) + # Note: we validate the following expected warning here, to avoid printing it. + with self.assertWarns(Warning) as warning_assessor: + sp3_df_no_offline_removal = sp3.clean_sp3_orb(sp3_df, False) + + captured_warnings = warning_assessor.warnings + self.assertIn( + "Failed to grab filename from sp3 dataframe for error output purposes:", str(captured_warnings[0].message) + ) + self.assertEqual( + len(captured_warnings), + 1, + "Only expected one warning, about failing to get path. Check what other warnings are being raised!", + ) self.assertTrue( np.array_equal(sp3_df_no_offline_removal.index.get_level_values(0).unique(), [774619200]), @@ -441,12 +477,27 @@ def test_clean_sp3_orb(self): "After cleaning there should be no dupe PRNs. As offline sat removal is off, offline sat should remain", ) - # Now check with offline sat removal enabled too - sp3_df_with_offline_removal = sp3.clean_sp3_orb(sp3_df, True) - # Check that we still seem to have one epoch with no dupe sats, and now with the offline sat removed - self.assertTrue( - np.array_equal(sp3_df_with_offline_removal.index.get_level_values(1), ["G01", "G02"]), - "After cleaning there should be no dupe PRNs (and with offline removal, offline sat should be gone)", + # NOTE: for some inexplicable reason, the following invocation doesn't seem to print its warning to the + # terminal when run in unittest's single file mode. I.e: + # - python -m unittest discover -v --> **Warning raised and printed** + # - python -m unittest test_sp3.py -v --> **Warning raised (assertWarns() is successful), but not printed** + + with self.assertWarns(Warning) as warning_assessor: + # Now check with offline sat removal enabled too + sp3_df_with_offline_removal = sp3.clean_sp3_orb(sp3_df, True) + # Check that we still seem to have one epoch with no dupe sats, and now with the offline sat removed + self.assertTrue( + np.array_equal(sp3_df_with_offline_removal.index.get_level_values(1), ["G01", "G02"]), + "After cleaning there should be no dupe PRNs (and with offline removal, offline sat should be gone)", + ) + captured_warnings = warning_assessor.warnings + self.assertIn( + "Failed to grab filename from sp3 dataframe for error output purposes:", str(captured_warnings[0].message) + ) + self.assertEqual( + len(captured_warnings), + 1, + "Only expected one warning, about failing to get path. Check what other warnings are being raised!", ) def test_gen_sp3_fundamentals(self): @@ -932,19 +983,26 @@ def test_sp3_comment_append_and_overwrite(self): def test_gen_sp3_content_velocity_exception_handling(self): """ - gen_sp3_content() velocity output should raise exception (currently unsupported).\ - If asked to continue with warning, it should remove velocity columns before output. + gen_sp3_content() can't yet output velocity data. Ensure raises by default, and removes vel columns with warning """ # Input data passed as bytes here, rather than using a mock file, because the mock file setup seems to break # part of Pandas Styler, which is used by gen_sp3_content(). Specifically, some part of Styler's attempt to # load style config files leads to a crash, despite some style config files appearing to read successfully) input_data_fresh = input_data + b"" # Lazy attempt at not passing a reference sp3_df = sp3.read_sp3(bytes(input_data_fresh), pOnly=False) + with self.assertRaises(NotImplementedError): generated_sp3_content = sp3.gen_sp3_content(sp3_df, continue_on_unhandled_velocity_data=False) - generated_sp3_content = sp3.gen_sp3_content(sp3_df, continue_on_unhandled_velocity_data=True) - self.assertTrue("VX" not in generated_sp3_content, "Velocity data should be removed before outputting SP3") + with self.assertWarns(Warning) as warning_accessor: + generated_sp3_content = sp3.gen_sp3_content(sp3_df, continue_on_unhandled_velocity_data=True) + self.assertTrue("VX" not in generated_sp3_content, "Velocity data should be removed before outputting SP3") + + captured_warnings = warning_accessor.warnings + self.assertEqual( + "SP3 velocity output not currently supported! Dropping velocity columns before writing out.", + str(captured_warnings[0].message), + ) def test_sp3_clock_nodata_to_nan(self): sp3_df = pd.DataFrame({("EST", "CLK"): [999999.999999, 123456.789, 999999.999999, 987654.321]})