Merge pull request #13 from userjack6880/dev

Dev

Author: userjack6880

.github/ISSUE_TEMPLATE/bug_report.md

@@ -24,6 +24,8 @@ A clear and concise description of what you expected to happen.
 If applicable, add screenshots to help explain your problem.
 
 **Software Version**
+[] Version 0 Alpha 5
+[] Version 0 Alpha 4
 [] Version 0 Alpha 3
 [] Version 0 Alpha 2
 

.github/ISSUE_TEMPLATE/critical_report.md

@@ -24,6 +24,8 @@ A clear and concise description of what you expected to happen.
 If applicable, add screenshots to help explain your problem.
 
 **Software Version**
+[] Version 0 Alpha 5
+[] Version 0 Alpha 4
 [] Version 0 Alpha 3
 [] Version 0 Alpha 2
 [] Other

README.md

@@ -8,7 +8,7 @@ A Perl based tool to parse DMARC reports, based on John Levine's [rddmarc](http:
 
 Open Report Parser is a fork of [techsneeze's dmarcts-report-parser](https://github.com/techsneeze/dmarcts-report-parser), and was forked to more closely match the needs of [Open DMARC Analyzer](https://github.com/userjack6880/Open-DMARC-Analyzer).
 
-Open Report Parser Version 0 Alpha 4 (0-α4) is an [Anomaly \<Codebase\>](https://systemanomaly.com/codebase) project by John Bradley ([email protected]).
+Open Report Parser Version 0 Alpha 5 (0-α5) is an [Anomaly \<Codebase\>](https://systemanomaly.com/codebase) project by John Bradley ([email protected]).
 
 # Minimum Requirements
 
@@ -224,9 +224,9 @@ Currently, processing of both DMARC and TLS reports during the same run is only
 
 # Latest Changes
 
-## 0-α4
-- Fixed Issue #8 again - some errornous code was introduced at some point that skips DB checks if not using Oauth2.
-- Updated documentation to fix minimum requirements regarding MariaDB vs MySQL (initially discovered with Open Report Analyzer).
+## 0-α5
+- Fixed an issue with undefined variables discovered with Issue #12.
+- Fixed an issue with mimecast improperly setting mime type outlined in Issue #12.
 
 # Tested System Configurations
 | OS          | Perl      | SQL             | Source                          |
@@ -244,9 +244,9 @@ Support will be provided as outlined in the following schedule. For more details
 
 | Version                             | Support Level    | Released         | End of Support   | End of Life      |
 | ----------------------------------- | ---------------- | ---------------- | ---------------- | ---------------- |
-| Version 1 Alpha 4                   | Full Support     | 8 June 2023      | TBD              | TBD              |
-| Version 1 Alpha 3                   | Critical Support | 25 May 2023      | 8 June 2023      | TBD              |
-| Version 1 Alpha 2                   | End of LIfe      | 26 April 2023    | 25 May 2023      | 8 June 2023      |
+| Version 1 Alpha 5                   | Full Support     | 10 July 2023     | TBD              | TBD              |
+| Version 1 Alpha 4                   | Critical Support | 8 June 2023      | 10 July 2023     | TBD              |
+| Version 1 Alpha 3                   | End of Life      | 25 May 2023      | 8 June 2023      | 10 July 2023     |
 
 # Contributing
 

docs/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0-α5
+- Fixed an issue with undefined variables discovered with Issue #12.
+- Fixed an issue with mimecast improperly setting mime type outlined in Issue #12.
+
 ## 0-α4
 - Fixed Issue #8 again - some errornous code was introduced at some point that skips DB checks if not using Oauth2.
 - Updated documentation to fix minimum requirements regarding MariaDB vs MySQL (initially discovered with Open Report Analyzer).

docs/SECURITY.md

@@ -20,7 +20,8 @@ Open DMARC Analyzer Version Security Support
 
 | Version                    | Supported | End of Life      |
 | -------------------------- | --------- | ---------------- |
+| Version 0 Alpha 5          | YES       | TBD              |
 | Version 0 Alpha 4          | YES       | TBD              |
-| Version 0 Alpha 3          | YES       | TBD              |
+| Version 0 Alpha 3          | NO        | 10 July 2023     |
 | Version 0 Alpha 2          | NO        | 8 June 2023      |
 | Version 0 Alpha 1          | NO        | 25 May 2023      |

docs/SUPPORT.md

@@ -4,8 +4,9 @@ Due to the limited time available to developrs to maintain support, dates must b
 
 | Version                             | Support Level    | Released         | End of Support   | End of Life      |
 | ----------------------------------- | ---------------- | ---------------- | ---------------- | ---------------- |
-| Version 1 Alpha 4                   | Full Support     | 8 June 2023      | TBD              | TBD              |
-| Version 1 Alpha 3                   | Critical Support | 25 May 2023      | 8 June 2023      | TBD              |
+| Version 1 Alpha 5                   | Full Support     | 10 July 2023     | TBD              | TBD              |
+| Version 1 Alpha 4                   | Critical Support | 8 June 2023      | 10 July 2023     | TBD              |
+| Version 1 Alpha 3                   | End of Life      | 25 May 2023      | 8 June 2023      | 10 July 2023     |
 | Version 1 Alpha 2                   | End of LIfe      | 26 April 2023    | 25 May 2023      | 8 June 2023      |
 | Version 1 Alpha 1                   | End of Life      | 19 April 2023    | 26 April 2023    | 25 May 2023      |
 

report-parser.pl

@@ -144,10 +144,13 @@ sub show_usage {
 $reports_replace = 0;
 $imapauth        = 'simple';
 $clear_token     = 0;
+$processInfo     = 0;
+$oauthuri        = '';
+$oauthclientid   = '';
 
 # used in messages
 my $scriptname = 'Open Report Parser';
-my $version    = 'Version 0 Alpha 4';
+my $version    = 'Version 0 Alpha 5';
 
 # allowed values for the DB columns, also used to build the enum() in the
 # CREATE TABLE statements in checkDatabase(), in order defined here
@@ -895,7 +898,31 @@ sub getXMLFromZip {
     my $unzip = "";
     if ($isgzip) {
       open(XML, "<:gzip", $filename)
-      or $unzip = "ungzip";
+      or $unzip = "gzip";
+
+      # check to see if the mimetype was incorrectly set to gzip *cough mimecast*
+      if ($unzip eq "gzip") {
+        printDebug("gzip failed, checking if incorrect mime type");
+        # incorrect mimetype with incorrect file extension seems to manifest as a .dat
+        # so let's check for that
+        my $ext = '';
+        if ($filename =~ /\.([^.]+)$/) {
+          print "extension: $1" if $debug;
+          $ext = $1;
+        }
+        else {
+          warn "$scriptname: Could not find extension for (<$filename>)! \n";
+        }
+
+        # if it's a dat file, then run this process again, but with $isgzip set to 0
+        if ($ext eq "dat") {
+          printDebug("Trying again using unzip");
+          $xml = getXMLFromZip($filename,0);
+
+          # if it fails again, it'll warn, if not, it'll return the correct xml
+          return $xml;
+        }
+      }
     } 
     else {
       open(XML, "-|", "unzip", "-p", $filename)
@@ -967,18 +994,18 @@ sub storeXMLInDatabase {
   my $policy_aspf  = undef;
   my $policy_p     = undef;
   my $policy_sp    = undef;
-  my $policy_pct     = undef;
+  my $policy_pct   = undef;
 
   if (ref $xml->{'policy_published'} eq "HASH") {
-    $domain       =  $xml->{'policy_published'}->{'domain'};
+    $domain       = $xml->{'policy_published'}->{'domain'};
     $policy_adkim = $xml->{'policy_published'}->{'adkim'};
     $policy_aspf  = $xml->{'policy_published'}->{'aspf'};
     $policy_p     = $xml->{'policy_published'}->{'p'};
     $policy_sp    = $xml->{'policy_published'}->{'sp'};
     $policy_pct   = $xml->{'policy_published'}->{'pct'};
   } 
   else {
-    $domain       =  $xml->{'policy_published'}[0]->{'domain'};
+    $domain       = $xml->{'policy_published'}[0]->{'domain'};
     $policy_adkim = $xml->{'policy_published'}[0]->{'adkim'};
     $policy_aspf  = $xml->{'policy_published'}[0]->{'aspf'};
     $policy_p     = $xml->{'policy_published'}[0]->{'p'};
@@ -1419,7 +1446,31 @@ sub getJSONFromZip {
     my $unzip = "";
     if ($isgzip) {
       open(JSON, "<:gzip", $filename)
-      or $unzip = "ungzip";
+      or $unzip = "gzip";
+
+      # check to see if the mimetype was incorrectly set to gzip *cough mimecast*
+      if ($unzip eq "gzip") {
+        printDebug("gzip failed, checking if incorrect mime type");
+        # incorrect mimetype with incorrect file extension seems to manifest as a .dat
+        # so let's check for that
+        my $ext = '';
+        if ($filename =~ /\.([^.]+)$/) {
+          print "extension: $1" if $debug;
+          $ext = $1;
+        }
+        else {
+          warn "$scriptname: Could not find extension for (<$filename>)! \n";
+        }
+
+        # if it's a dat file, then run this process again, but with $isgzip set to 0
+        if ($ext eq "dat") {
+          printDebug("Trying again using unzip");
+          $json = getJSONFromZip($filename,0);
+
+          # if it fails again, it'll warn, if not, it'll return the correct xml
+          return $json;
+        }
+      }
     } 
     else {
       open(JSON, "-|", "unzip", "-p", $filename)