Changed 'wait time' for retries to increase depending on number of attempts made to load page. (#17)

Author: ReidWeb

CHANGELOG.md

@@ -1,3 +1,7 @@
+v1.3.4
+======
+* Changed 'wait time' for retries to increase depending on number of attempts made to load page.
+
 v1.3.3
 ======
 * If the Gatherer experiences a HTTP error code `429` (Too many requests), it will pause for 1-20 milliseconds and retry via  recursive method call.

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.ffxivcensus.gatherer</groupId>
     <artifactId>XIVStats-Gatherer-Java</artifactId>
-    <version>v1.3.3</version>
+    <version>v1.3.4</version>
     <name>XIVStats Lodestone Gatherer</name>
     <url>https://github.com/xivstats</url>
 

src/main/java/com/ffxivcensus/gatherer/Gatherer.java

@@ -35,7 +35,7 @@ public void run() {
                     System.out.println("Starting evaluation of player ID: " +nextID);
                 }
                 //Parse players and write them to DB
-                String out = parent.writeToDB(Player.getPlayer(nextID));
+                String out = parent.writeToDB(Player.getPlayer(nextID, 1));
                 if (!parent.isQuiet()) { //If not running in quiet mode
                     System.out.println(out);
                 }
@@ -49,7 +49,7 @@ public void run() {
                 }
                 //Then attempt to write again
                 try {
-                    String out = parent.writeToDB(Player.getPlayer(nextID));
+                    String out = parent.writeToDB(Player.getPlayer(nextID,1));
                     if (!parent.isQuiet()) {
                         System.out.println(out);
                     }
@@ -68,4 +68,3 @@ public void run() {
 
 }
 
-

src/main/java/com/ffxivcensus/gatherer/Player.java

@@ -1853,10 +1853,11 @@ public boolean doesPlayerHaveMinion(String minionName) {
      * Fetch a player from the lodestone specified by ID.
      *
      * @param playerID the ID of the player to fetch
+     * @param attempt the number of times this character has been attempted.
      * @return the player object matching the specified ID.
      * @throws Exception exception thrown if more class levels returned than anticipated.
      */
-    public static Player getPlayer(int playerID) throws Exception {
+    public static Player getPlayer(int playerID, int attempt) throws Exception {
         //Initialize player object to return
         Player player = new Player(playerID);
         //Declare HTML document
@@ -1923,13 +1924,14 @@ public static Player getPlayer(int playerID) throws Exception {
             String strEx = org.apache.commons.lang.exception.ExceptionUtils.getStackTrace(ioEx);
             String statusCode = strEx.split("\\s+")[5].replace("Status=","").replace(",","");
             if(statusCode.equals("429")) {
-                //Generate random number 1-20 and sleep for it
+                //Generate random number 1->20*attempt no and sleep for it
                 Random rand = new Random();
-
-                int randomNum = rand.nextInt((20 - 1) + 1) + 1;
-                System.out.println("Experiencing rate limiting (HTTP 429) while fetching id " + playerID + ", waiting " + randomNum + "ms then retrying...");
+                int max = attempt * 20;
+                int min = (attempt - 1) + 1;
+                int randomNum = rand.nextInt(max - min + 1) + min;
+                System.out.println("Experiencing rate limiting (HTTP 429) while fetching id " + playerID + " (attempt " + attempt + "), waiting " + randomNum + "ms then retrying...");
                 TimeUnit.MILLISECONDS.sleep(randomNum);
-                player = Player.getPlayer(playerID);
+                player = Player.getPlayer(playerID, ++attempt);
             } else {
                 throw new Exception("Character " + playerID + " does not exist. Status: " + statusCode);
             }
@@ -2423,4 +2425,4 @@ public int getBitHasVath() {
     public void setHasVath(boolean hasVath) {
         this.hasVath = hasVath;
     }
-}
+}

src/test/java/com/ffxivcensus/gatherer/PlayerTest.java

@@ -21,7 +21,7 @@ public class PlayerTest {
     @org.junit.Test
     public void testGetPlayer() throws Exception {
         //Fetch object to test against (Aelia Sokoto, Cerberus)
-        Player playerOne = Player.getPlayer(2356533);
+        Player playerOne = Player.getPlayer(2356533,1);
 
         //NOTE: All of the following tests assume various pieces of information
         //Testing information that is very unlikely to change
@@ -167,7 +167,7 @@ public void testGetPlayer() throws Exception {
      */
     @org.junit.Test
     public void testGetVeteranPlayer() throws Exception {
-        Player player = Player.getPlayer(501646);
+        Player player = Player.getPlayer(501646,1);
 
         //Player has 960 days sub, make sure recorded correctly
         assertTrue(player.isHas960DaysSub());
@@ -187,7 +187,7 @@ public void testGetVeteranPlayer() throws Exception {
      */
     @org.junit.Test
     public void testUnplayedPlayer() throws Exception {
-        Player player = Player.getPlayer(13002142);
+        Player player = Player.getPlayer(13002142,1);
 
         //Test grand company
         assertEquals("none", player.getGrandCompany());
@@ -240,7 +240,7 @@ public void testUnplayedPlayer() throws Exception {
      */
     @org.junit.Test
     public void testGetPlayerNoGCHasFC() throws Exception {
-        Player player = Player.getPlayer(1);
+        Player player = Player.getPlayer(1,1);
 
         //Verify that grand company is "None"
         assertEquals("none", player.getGrandCompany());
@@ -254,7 +254,7 @@ public void testGetPlayerNoGCHasFC() throws Exception {
      */
     @org.junit.Test
     public void testGetPlayerNoFCHasGC() throws Exception {
-        Player player = Player.getPlayer(11886920);
+        Player player = Player.getPlayer(11886920,1);
 
         //Test that GC is maelstrom
         assertEquals("Maelstrom", player.getGrandCompany());
@@ -270,7 +270,7 @@ public void testGetPlayerNoFCHasGC() throws Exception {
      */
     @org.junit.Test
     public void testGetPlayerWithAllCollectibles() throws Exception {
-        Player player = Player.getPlayer(71);
+        Player player = Player.getPlayer(71,1);
 
         assertEquals(player.getBitHasArrArtbook(), 1);
         assertEquals(player.getBitHasBeforeMeteor(), 1);
@@ -287,12 +287,12 @@ public void testGetPlayerWithAllCollectibles() throws Exception {
     public void testGetPlayerInvalid() {
         try {
             //Try to get a character that doesn't exist
-            Player player = Player.getPlayer(2356539);
+            Player player = Player.getPlayer(2356539,1);
             //If no exception thrown fail
             fail("Character should not exist");
         } catch (Exception e) {
 
         }
     }
 
-}
+}