count2txt

#!/usr/bin/env perl 
#===============================================================================
#
#         FILE: count2txt
#
#        USAGE: count2txt <file1.csv file2.CSV ...>
#
#      OPTIONS: -n, --dry-run   read files and count words, but write no output
#               --multifile     write one .txt file per CSV file
#               -q, --quiet     no log messages (warnings still enabled)
#
#  DESCRIPTION: The opposite of compression.
#
#  Turn JSTOR DFR wordcount files in CSV format, *.CSV, into bags of
#  words. That way you can feed original documents to any tool that
#  wants them---though the documents will be...slightly...scrambled.
#
#  By default, each input file is output to stdout as a single line of
#  the form
#
#  <filename> word word word word word word ...
#
#  This output can be redirected to a file and fed to mallet import-file.
#
#  With --multifile, each input file <file.csv> is output to a corresponding
#  text file <file.txt>. The aggregate of such files can be fed to
#  mallet import-dir.
#
#       AUTHOR: Andrew Goldstone (agoldst), andrew.goldstone@gmail.com
# ORGANIZATION: Rutgers University, New Brunswick
#      VERSION: 0.1 
# REQUIREMENTS: perl 5.14. Uses only modules that should be part of a
# basic perl installation. Should work fine with earlier perls, but
# haven't tested this. It works with my copy of perl 5.12 (changing the
# "use v5.14" pragma, of course).
# 
#     WARNING: use at your own risk. Amateur hour. Note that running
# this on a lot of files means generating a lot of data. Make sure you
# have space on your hard drive. This is not the right way to prep
# jstor's wordcount data for topic modeling or any other analysis, just
# a stopgap I am using.  
#
#===============================================================================
use v5.14;
use utf8;                                   # source code itself is in utf-8
use warnings;
use warnings FATAL => "utf8";               # Unicode encode errors are fatal
use autodie;
use open qw( :std :utf8 );                  # default utf8 layer

use File::Basename;

# Command-line options processing

use Getopt::Long;

my $dry_run="";
my $multifile="";
my $QUIET_LOG="";
GetOptions(
    "dry-run|n" => \$dry_run,
    "multifile"  => \$multifile,
    "quiet|n"  => \$QUIET_LOG,
    );

log_msg("Dry run: no files will be written\n") if $dry_run;

# Output filehandle. We'll set it according to $multiline option
my $fh;

unless($multifile) {
    $fh = \*STDOUT;
}

foreach my $csv_file (@ARGV) {
    unless( -f $csv_file ) {
        warn "$csv_file not found; skipping\n";
        next;
    }
    unless( $csv_file =~ /\.csv/i ) {
        warn "$csv_file is not of form *.(csv|CSV); skipping\n";
        next;
    }
    open INFILE, "<:encoding(UTF-8)", $csv_file
        or die "Couldn't open $csv_file for reading";


    # Split up the filename into directory path, name, csv suffix
    my ($stem,$directories,$suffix) = fileparse($csv_file,qr/\.csv/i);

    # for multifile output, open up the new *.txt

    if($multifile) {
        my $filename = "${directories}$stem.txt"; 

        if( -f $filename ) {
            warn "Output $filename already exists; skipping processing of $csv_file\n";
            next;
        }

        unless($dry_run) {
            open $fh, ">:encoding(UTF-8)", "$filename"
                or die "Couldn't open $filename for writing";
        }
    }

    # double-check that this input file has the expected header
    # jstor calls the count "weight" because you can actually write jstor
    # search queries that specify weightings
    my $header = <INFILE>;
    unless($header && $header eq "WORDCOUNTS,WEIGHT\n") {
        warn "unexpected header found: $header\nSkipping $csv_file\n";
        next;
    }

    # single file output: the start of a line is the name of the file
    # we're turning into a bag of words

    unless($multifile) {
        print "${stem}$suffix ";
    }

    my $total_count = 0;
    while(<INFILE>) {
        chomp;
        my ($word,$count) = split /,/;

        unless($dry_run) { 
            print $fh "$word " x (0 + $count); # I love the x operator
        }

        # keep track of the total number of words to provide sanity check
        # and running feedback

        $total_count += $count;
    }
    print $fh "\n";

    log_msg("$csv_file: $total_count words written\n");

    # clean up before the next iteration of the loop

    close INFILE or die "Couldn't close input file";

    if(!$dry_run && $multifile) { 
        close $fh
            or die "Couldn't close output file";
    }
}

# basic logging

sub log_msg {
    print STDERR @_ unless $QUIET_LOG;
}