dmdstats

#!/usr/bin/env perl

package dmdstats;

use strict;
use warnings;
use FindBin;
use lib "$FindBin::RealBin/../lib";
use Getopt::Long;
use Fcntl qw(:DEFAULT :flock);
use Try::Tiny;
use Log::Log4perl;
use File::Find;
use Data::Dumper;
use Digest::MD5;
use JSON;
use URI::Escape;
use DateTime::Format::ISO8601;
use HTTP::Date qw(:DEFAULT time2isoz);
use File::Path qw(make_path remove_tree);
use File::Spec;
use File::stat;
use File::chdir;
use utf8;
use Switch;
use Text::CSV qw( csv );
use Archive::Zip qw( :ERROR_CODES :CONSTANTS );

# There are only 3 valid types of descriptive metadata
my @dmdtypes = ( "issueinfo", "DC", "MARC" );

Log::Log4perl->init_once("/etc/canadiana/tdr/log4perl.conf");
my $logger = Log::Log4perl::get_logger("CIHM::TDR");

sub log_warnings {
    my $warning = shift;
    chomp $warning;

    # Strip wide characters before  trying to log
    ( my $stripped = $warning ) =~ s/[^\x00-\x7f]//g;

    $logger->warn($stripped);
    print STDERR "$warning\n";
}
local $SIG{__WARN__} = sub { &log_warnings };

my $dmddir = '/crkn-nas-wip/_Metadata_Synchronised';
$dmddir = $ENV{syncdmd_dmddir} if ( exists $ENV{syncdmd_dmddir} );
my $outfile = File::Spec->catfile( $dmddir, 'analysis/dmdstats.csv' );
my $outdir  = File::Spec->catfile( $dmddir, 'analysis/dmdstats-IDlists/' );
my $zipdir  = File::Spec->catfile( $dmddir, 'dmdZIP/' );

GetOptions(
    'outfile:s' => \$outfile,
    'outdir:s'  => \$outdir,
    'zipdir:s'  => \$zipdir,
    'dmddir:s'  => \$dmddir,
);

# Make sure a clean output directory exists.
make_path($outdir);
remove_tree( $outdir, { safe => 1, keep_root => 1, verbose => 1 } );

# Make sure a clean output directory exists.
make_path($zipdir);
remove_tree( $zipdir, { safe => 1, keep_root => 1, verbose => 1 } );

$logger->info("DMD stats: start");

# Used to show a different processname during processing
my $dmdstatsprog = $0;

my %results;
foreach my $repository ( "access", "preservation" ) {
    my $scandir = File::Spec->catfile( $dmddir, $repository );
    find( \&matching_dmd_file, $scandir );
}

sub matching_dmd_file {
    if ( -f $_ ) {
        $0 = $dmdstatsprog . " counting in $File::Find::dir";
        if (/^(.*)-(\w*)\.xml$/) {
            my $id      = $1;
            my $dmdType = $2;

            if ( $File::Find::dir =~ /$dmddir\/([^\/]+)\/(.*)$/ ) {
                my $repo   = $1;
                my $prefix = $2;

                if ( !( defined $results{$repo}{$prefix}{$dmdType} ) ) {
                    $results{$repo}{$prefix}{$dmdType} = [];
                }
                push @{ $results{$repo}{$prefix}{$dmdType} }, $id;
            }
            else {
                warn $File::Find::dir . "  -- doesn't match pattern\n";
            }
        }
        else {
            warn $_ . "  -- doesn't match pattern (in $File::Find::dir )\n";
        }
    }
}

my @res = [ "repository", "prefix" ];
push @{ $res[0] }, sort @dmdtypes, "totals";

foreach my $repo ( sort keys %results ) {
    foreach my $prefix ( sort keys %{ $results{$repo} } ) {
        my @thisline = ( $repo, $prefix );
        my $total = 0;
        foreach my $dmdType ( sort @dmdtypes ) {
            if ( defined $results{$repo}{$prefix}{$dmdType} ) {
                my @aipids = @{ $results{$repo}{$prefix}{$dmdType} };
                my $count  = scalar(@aipids);

                push @thisline, $count;
                $total += $count;

                my $outfile =
                  File::Spec->catfile( $outdir, "$repo-$prefix-$dmdType.csv" );
                $0 = $dmdstatsprog . " writing $outfile";

                open my $fh, '>', $outfile or die "Cannot open $outfile: $!";
                foreach (@aipids) {
                    print $fh "$_\n";
                }
                close $fh;
            }
            else {
                push @thisline, 0;
            }
        }
        push @thisline, $total;
        push @res, [@thisline];
    }
}

csv( in => \@res, out => $outfile );

foreach my $repo ( sort keys %results ) {
    foreach my $prefix ( sort keys %{ $results{$repo} } ) {
        local $CWD = File::Spec->catfile( $dmddir, $repo, $prefix );

        foreach my $dmdType ( sort @dmdtypes ) {
            if ( defined $results{$repo}{$prefix}{$dmdType} ) {
                my $zipfile =
                  File::Spec->catfile( $zipdir, "$repo-$prefix-$dmdType.zip" );
                $0 = $dmdstatsprog . " creating $zipfile";

                my $zip = Archive::Zip->new();
                foreach my $id ( @{ $results{$repo}{$prefix}{$dmdType} } ) {
                    my $xmlfile = $id . "-" . $dmdType . ".xml";
                    if ( -f $xmlfile ) {
                        $zip->addFile($xmlfile);
                    }
                    else {
                        warn "$xmlfile not found\n";
                    }
                }
                $0 = $dmdstatsprog . " writing $zipfile";

                # Save the Zip file
                unless ( $zip->writeToFileNamed($zipfile) == AZ_OK ) {
                    die 'write error';
                }
            }
        }
    }
}