Merge pull request #3 from LCAS/copilot/fix-json-decode-error

Add response validation before JSON parsing to prevent decode errors

Author: marc-hanheide

figshare.py

@@ -53,8 +53,14 @@ def shorten(self, doi):
             quoted_doi = urllib.request.quote(doi)
             url = 'http://shortdoi.org/{}?format=json'.format(quoted_doi)
             try:
-                response = requests.get(url).json()
-                short_doi = response['ShortDOI']
+                response = requests.get(url)
+                # Check if response is valid and contains JSON
+                if response.ok and response.headers.get('Content-Type', '').lower().startswith('application/json') and response.text.strip():
+                    result = response.json()
+                    short_doi = result['ShortDOI']
+                else:
+                    self.logger.warning(f"Received empty or invalid JSON response for {doi} from {url} (status: {response.status_code})")
+                    return None
             except Exception as e:
                 self.logger.warning(f"failed to get short doi for {doi}: {e}")
                 return None
@@ -150,21 +156,33 @@ def __get(self, url, params=None, use_cache=True):
             return self.__cache[hash_key]
         else:
             headers = { "Authorization": "token " + self.token } if self.token else {}
-            result = get(self.base_url + url, headers=headers, params=params).json()
-            self.__cache[hash_key] = result
-            self.save_cache()
-            return result
+            response = get(self.base_url + url, headers=headers, params=params)
+            # Check if response is valid and contains JSON
+            if response.ok and response.headers.get('Content-Type', '').lower().startswith('application/json') and response.text.strip():
+                result = response.json()
+                self.__cache[hash_key] = result
+                self.save_cache()
+                return result
+            else:
+                self.logger.warning(f"Received empty or invalid JSON response for GET {self.base_url + url} (status: {response.status_code})")
+                return {}
 
     def __post(self, url, params=None, use_cache=True):
         hash_key = f"POST{url}?{params}"
         if hash_key in self.__cache and use_cache:
             return self.__cache[hash_key]
         else:
             headers = { "Authorization": "token " + self.token } if self.token else {}
-            result = post(self.base_url + url, headers=headers, json=params).json()
-            self.__cache[hash_key] = result
-            self.save_cache()
-            return result
+            response = post(self.base_url + url, headers=headers, json=params)
+            # Check if response is valid and contains JSON
+            if response.ok and response.headers.get('Content-Type', '').lower().startswith('application/json') and response.text.strip():
+                result = response.json()
+                self.__cache[hash_key] = result
+                self.save_cache()
+                return result
+            else:
+                self.logger.warning(f"Received empty or invalid JSON response for POST {self.base_url + url} (status: {response.status_code})")
+                return []
 
 
     def articles_by_user_name(self, user_name, use_cache=True):
@@ -276,7 +294,13 @@ def _guess_doi(self, article):
                 self.logger.debug(f"Querying Crossref for title: {title}")
                 response = requests.get(base_url, params=params)
                 response.raise_for_status()
-                data = response.json()
+                
+                # Check if response is valid and contains JSON
+                if response.ok and response.headers.get('Content-Type', '').lower().startswith('application/json') and response.text.strip():
+                    data = response.json()
+                else:
+                    self.logger.warning(f"Received empty or invalid JSON response from Crossref API (status: {response.status_code})")
+                    return None
 
                 if data["message"]["total-results"] == 0:
                     self.logger.debug(f"No DOI found for: {title}")