utils/moz.pl at master · clamey/utils · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
#!/usr/bin/perl

use warnings;
use strict;
use Text::CSV;
use JSON qw( decode_json );
use Data::Dumper;
use URI::Escape;
require LWP::UserAgent;

my @rows;
my $incsv = Text::CSV->new ( { binary => 1 } ) or die "Cannot use CSV: ".Text::CSV->error_diag ();
my $outcsv = Text::CSV->new ( { binary => 1 } ) or die "Cannot use CSV: ".Text::CSV->error_diag ();

open my $fh, "<:encoding(utf8)", "orig.txt" or die "orig.txt$!";

push @rows, [ 'url','closely_id','name','location','score','potential_score','%_incomplete','%_inconsistent','%_duplicate','missing' ];

my $cnt = 0;
my $ua = LWP::UserAgent->new;
open my $ofh, ">:encoding(utf8)", "biz.csv" or die "biz.csv: $!";

while ( my $row = $incsv->getline( $fh ) ) {
  $cnt++;
  my @newrow;

  my $name = uri_escape_utf8($row->[1]);
  my $location = $row->[5];
  # my $location = uri_escape_utf8($row->[2] . " " . $row->[3] . " " . $row->[4] . " " . $row->[5]);
  my $url = "https://moz.com/local/api/perch?q=$name&loc=$location";

  # print "Hitting $url\n";

  my $response;
  if ($name && $location) {
    $response = $ua->get($url);
  }

  push @newrow, $url;
  push @newrow, $row->[0];
  push @newrow, $row->[1];
  push @newrow, $location;

  if ($location && $name && $response && $response->is_success && $response->content
    && $response->content_length && $response->content_length > 0) {


    print STDERR "\n\nJSON :\n\n";
    print STDERR Dumper $response->content;
    print STDERR "\n\n";

    my %json = %{ decode_json($response->content) };
    if (%json && (scalar keys %json > 0)) {
      push @newrow, $json{score};
      push @newrow, $json{potential}{score};
      push @newrow, $json{potential}{incomplete};
      push @newrow, $json{potential}{inconsistent};
      push @newrow, $json{potential}{duplicates};

      if ($json{missing}) {
        my @missing = @{ $json{missing} };
        my $out;
        foreach my $item (@missing) {
          if ($out) {
            $out .= "|";
          }
          $out .= ${$item}{source};
        }

        push @newrow, $out;
      }
    }
  }
  else {
    print STDERR "ERROR\n\n\n" . $url . "\n\n\n\n";
    if ($response) {
      print STDERR Dumper $response . "\n\n\n";
    }
  }

  push @rows, \@newrow;
  if ($cnt % 10 == 0) {
    print "Count at $cnt\n";
    $outcsv->eol ("\r\n");
    $outcsv->print ($ofh, $_) for @rows;
    $ofh->flush();
    @rows = [];
  }
}

$incsv->eof or $incsv->error_diag();
close $fh;


$outcsv->eof or $outcsv->error_diag();
close $ofh or die "biz.csv: $!";