handle corrupted replays

Author: howardchung

pom.xml

@@ -24,26 +24,8 @@
             <id>jitpack.io</id>
             <url>https://jitpack.io</url>
         </repository>
-        <repository>
-            <id>sonatype.oss.snapshots</id>
-            <name>Sonatype OSS Snapshot Repository</name>
-            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
-            <releases>
-                <enabled>false</enabled>
-            </releases>
-            <snapshots>
-                <enabled>true</enabled>
-            </snapshots>
-        </repository>
     </repositories>
     <dependencies>
-<!--
-        <dependency>
-            <groupId>com.github.skadistats</groupId>
-            <artifactId>clarity</artifactId>
-            <version>master-SNAPSHOT</version>
-        </dependency>
--->
 <!--
         <dependency>
             <groupId>com.github.skadistats</groupId>
@@ -62,16 +44,22 @@
             <artifactId>gson</artifactId>
             <version>2.8.9</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.28.0</version>
+        </dependency>
     </dependencies>
-      <dependencyManagement>
+    <!-- <dependencyManagement>
     <dependencies>
       <dependency>
         <groupId>com.skadistats</groupId>
         <artifactId>clarity-protobuf</artifactId>
         <version>5.4</version>
       </dependency>
     </dependencies>
-  </dependencyManagement>
+  </dependencyManagement> -->
     <build>
         <plugins>
             <plugin>

scripts/test.sh

@@ -4,4 +4,5 @@ curl localhost:5600 --data-binary "@/home/howardc/parser/8583844960_1679394786.d
 
 curl localhost:5600/blob?replay_url=http://replay271.valve.net/570/8623065989_1426557301.dem.bz2
 curl localhost:5600/blob?replay_url=http://www.example.com
-curl localhost:5600/blob?replay_url=https://odota.github.io/testfiles/1781962623_1.dem
+curl localhost:5600/blob?replay_url=https://odota.github.io/testfiles/1781962623_1.dem
+curl localhost:5600/blob?replay_url=http://replay113.valve.net/570/2562582896_1605238528.dem.bz2

src/main/java/opendota/Main.java

@@ -93,30 +93,42 @@ public void handle(HttpExchange t) throws IOException {
                 if (replayUrl.toString().endsWith(".bz2")) {
                     tStart = System.currentTimeMillis();
                     // Write byte[] to bunzip, get back decompressed byte[]
+                    // The C decompressor is a bit faster than Java, 4.3 vs 4.8s
+                    // BZip2CompressorInputStream bz = new BZip2CompressorInputStream(new ByteArrayInputStream(bzIn));
+                    // bzOut = bz.readAllBytes();
+                    // bz.close();
+
                     Process bz = new ProcessBuilder(new String[] { "bunzip2" }).start();
                     // Start separate thread so we can consume output while sending input
-                    Thread thread = new Thread(() -> {
+                    new Thread(() -> {
                         try {
                             bz.getOutputStream().write(bzIn);
                             bz.getOutputStream().close();
                         } catch (IOException ex) {
                             ex.printStackTrace();
                         }
-                    });
-                    thread.start();
+                    }).start();
 
                     bzOut = bz.getInputStream().readAllBytes();
+                    bz.getInputStream().close();
                     String bzError = new String(bz.getErrorStream().readAllBytes());
+                    bz.getErrorStream().close();
                     System.err.println(bzError);
+                    if (bzError.toString().contains("bunzip2: Data integrity error when decompressing") || bzError.contains("bunzip2: Compressed file ends unexpectedly")) {
+                        // Corrupted replay, don't retry
+                        t.sendResponseHeaders(200, 0);
+                        t.getResponseBody().close();
+                        return;
+                    }
                     tEnd = System.currentTimeMillis();
                     System.err.format("bunzip2: %dms\n", tEnd - tStart);
                 }
 
                 // Start parser with input stream created from byte[]
                 tStart = System.currentTimeMillis();
-                ByteArrayOutputStream baos2 = new ByteArrayOutputStream();
-                new Parse(new ByteArrayInputStream(bzOut), baos2, true);
-                byte[] parseOut = baos2.toByteArray();
+                ByteArrayOutputStream parseOutStream = new ByteArrayOutputStream();
+                new Parse(new ByteArrayInputStream(bzOut), parseOutStream, true);
+                byte[] parseOut = parseOutStream.toByteArray();
                 tEnd = System.currentTimeMillis();
                 System.err.format("parse: %dms\n", tEnd - tStart);