Skip to content

Commit f538edb

Browse files
markito3markito3
authored
Merge pull request #27 from JeffersonLab/disk_management
files to do survey of disk usage
2 parents f2e941f + 1a9d76b commit f538edb

File tree

2 files changed

+365
-0
lines changed

2 files changed

+365
-0
lines changed

‎disk_management/disk_database.pl‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
#!/usr/bin/env perl
2+
#
3+
# parameters
4+
#
5+
$starting_directory = $ARGV[0];
6+
$directory_label = $ARGV[1];
7+
if (! $starting_directory || !$directory_label) {
8+
print_usage_message();
9+
exit 1;
10+
}
11+
$dir_table = "${directory_label}_dir";
12+
$file_table = "${directory_label}_file";
13+
$update_time_table = "${directory_label}_updateTime";
14+
#
15+
# load perl modules
16+
#
17+
use DBI;
18+
#
19+
# connect to database
20+
#
21+
$host = 'localhost';
22+
$user = 'diskmanager';
23+
$password = '';
24+
$database = 'diskManagement';
25+
print "Connecting to $user\@$host, using $database.\n";
26+
$dbh_db = DBI->connect("DBI:mysql:$database:$host", $user, $password);
27+
if (defined $dbh_db) {
28+
print "Connection successful\n";
29+
} else {
30+
die "Could not connect to the database server, exiting.\n";
31+
}
32+
#
33+
# drop the tables
34+
#
35+
$sql = "drop table if exists $dir_table, $file_table, $update_time_table;";
36+
make_query($dbh_db, \$sth);
37+
#
38+
# re-create tables
39+
#
40+
$sql = "create table $dir_table (
41+
id int primary key auto_increment,
42+
dirname varchar(256),
43+
size int,
44+
uid smallint
45+
);";
46+
make_query($dbh_db, \$sth);
47+
$sql = "create table $file_table (
48+
id int primary key auto_increment,
49+
filename varchar(256),
50+
dirId int,
51+
atime datetime,
52+
size bigint,
53+
uid smallint
54+
);";
55+
make_query($dbh_db, \$sth);
56+
$sql = "create table $update_time_table (
57+
updateTime timestamp
58+
);";
59+
make_query($dbh_db, \$sth);
60+
$sql = "insert into $update_time_table set updateTime = NULL;";
61+
make_query($dbh_db, \$sth);
62+
#
63+
# collect the data
64+
#
65+
$find_dir_command = "find $starting_directory -xdev \\( \\(";
66+
$find_dir_command .= " -path $starting_directory/.snapshot";
67+
$find_dir_command .= " -o -path /volatile/halld/home/gxproj5/trackeff_ver99";
68+
$find_dir_command .= " -o -path /volatile/halld/home/jzarling/April18_bggen_omega/log";
69+
$find_dir_command .= " \\) -prune \\) -o \\( -type d -print \\) |";
70+
print "find_dir_command = $find_dir_command\n";
71+
open (FINDDIR, $find_dir_command);
72+
$idebug = 0;
73+
$jdebug = 0;
74+
while ($dirname = <FINDDIR>) {
75+
if (!($idebug%10)) {print "$idebug: dirname = $dirname";}
76+
chomp $dirname;
77+
@stat = stat($dirname);
78+
$sql = "insert into $dir_table set dirname = \"$dirname\", uid = $stat[4], size = $stat[7];";
79+
#print $sql, "\n";
80+
make_query($dbh_db, \$sth);
81+
$sql = "select last_insert_id();";
82+
make_query($dbh_db, \$sth);
83+
@row = $sth->fetchrow_array;
84+
$last_id = $row[0];
85+
$dirname =~ s/ /\\ /g; # escape space
86+
$dirname =~ s/\(/\\\(/g; # escape (
87+
$dirname =~ s/\)/\\\)/g; # escape )
88+
$dirname =~ s/&/\\&/g; # escape &
89+
$dirname =~ s/\$/\\\$/g; # escape $
90+
$dirname =~ s/>/\\>/g; # escape >
91+
$dirname =~ s/</\\</g; # escape <
92+
open (FINDFILE, "find $dirname -maxdepth 1 -type f |");
93+
while ($filename = <FINDFILE>) {
94+
chomp $filename;
95+
if (!($jdebug%100)) {print "$jdebug: $filename\n";}
96+
@stat = stat($filename);
97+
if (@stat) {
98+
@token = split(/\//, $filename);
99+
$file_no_path = $token[$#token];
100+
#print "file_no_path = $file_no_path\n";
101+
$file_no_path =~ s/\\/\\\\/g; # escape backslash
102+
$sql = "insert into $file_table set filename = \"$file_no_path\", dirId = $last_id, atime = from_unixtime($stat[8]), size = $stat[7], uid = $stat[4];";
103+
make_query($dbh_db, \$sth);
104+
} else {
105+
print "cannot stat $filename in $dirname\n";
106+
}
107+
$jdebug++;
108+
}
109+
close(FINDFILE);
110+
$idebug++;
111+
# print "DEBUG: $idebug directories done\n";
112+
# if ($idebug == 1000) {last;}
113+
}
114+
$sth = 0;
115+
#
116+
# disconnect from database server
117+
#
118+
print "disconnecting from server\n";
119+
$rc = $dbh_db->disconnect();
120+
print "disconnected\n";
121+
#
122+
# exit
123+
#
124+
print "exit time!\n";
125+
exit;
126+
#
127+
# make a query
128+
#
129+
sub make_query {
130+
131+
my($dbh, $sth_ref) = @_;
132+
$$sth_ref = $dbh->prepare($sql)
133+
or die "Can't prepare $sql: $dbh->errstr\n";
134+
135+
$rv = $$sth_ref->execute
136+
or die "Can't execute the query $sql\n error: $sth->errstr\n";
137+
138+
return 0;
139+
}
140+
141+
sub print_usage_message {
142+
print <<EOM;
143+
Usage:
144+
disk_database.pl <top level directory> <report label>
145+
EOM
146+
return;
147+
}
148+
#
149+
# end of file
150+
#

‎disk_management/disk_report.pl‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
#!/usr/bin/env perl
2+
3+
$directory_label = $ARGV[0];
4+
$dir_table = $directory_label . "_dir";
5+
$file_table = $directory_label . "_file";
6+
$update_time_table = $directory_label . "_updateTime";
7+
8+
use CGI; # load CGI routines
9+
use DBI;
10+
$user = "diskmanager";
11+
$password = "";
12+
$database = "diskManagement";
13+
$hostname = "localhost";
14+
15+
$seconds_per_year = 365.25*24*60*60;
16+
17+
$dbh = DBI->connect("DBI:mysql:$database:$hostname", $user, $password);
18+
19+
$q = new CGI; # create new CGI object
20+
$title = "Disk Usage Report: $directory_label";
21+
print
22+
$q->start_html($title), # start the HTML
23+
$q->h1($title), # level 1 header
24+
"<p>File ages are from last access time.\n";
25+
26+
$sql = "select updateTime from $update_time_table;";
27+
make_query($dbh, \$sth);
28+
@row = $sth->fetchrow_array;
29+
print "<p>Update time: $row[0]\n";
30+
31+
print $q->h2("Largest Files"), "\n";
32+
#############
33+
34+
$sql = "select $file_table.size, filename, atime, $file_table.uid, dirname from $dir_table, $file_table where $dir_table.id = dirId order by $file_table.size desc limit 10;";
35+
make_query($dbh, \$sth);
36+
print "<table border>\n";
37+
print "<tr><th>Rank<th>Size (GB)<th>File<th>Last Access Time<th>Owner<th>Directory\n";
38+
$i = 1;
39+
while (@row = $sth->fetchrow_array) {
40+
$size = round($row[0]/1.e9);
41+
$user = getpwuid($row[3]);
42+
if (! $user) {$user = "uid=$row[3]";}
43+
print "<tr><td>$i<td>$size<td>$row[1]<td>$row[2]<td>$user<td>$row[4]\n";
44+
$i++;
45+
}
46+
print "</table>\n";
47+
48+
print $q->h2("Oldest Files"), "\n";
49+
############
50+
51+
$sql = "select atime, filename, $file_table.size, $file_table.uid, dirname from $dir_table, $file_table where $dir_table.id = dirId order by atime limit 10;";
52+
make_query($dbh, \$sth);
53+
print "<table border>\n";
54+
print "<tr><th>Rank<th>Last Access Time<th>File<th>Size (GB)<th>Owner<th>Directory\n";
55+
$i = 1;
56+
while (@row = $sth->fetchrow_array) {
57+
$size = round($row[2]/1.e9);
58+
$user = getpwuid($row[3]);
59+
if (! $user) {$user = "uid=$row[3]";}
60+
print "<tr><td>$i<td>$row[0]<td>$row[1]<td>$size<td>$user<td>$row[4]\n";
61+
$i++;
62+
}
63+
print "</table>\n";
64+
65+
print $q->h2("Files with Greatest Size &times; Age"), "\n";
66+
####################################
67+
68+
$sql = "select ($file_table.size*1.e-9)*(unix_timestamp(now()) - unix_timestamp(atime)) as gby, filename, $file_table.size, atime, $dir_table.uid, dirname from $dir_table, $file_table where $dir_table.id = dirId order by gby desc limit 10;";
69+
make_query($dbh, \$sth);
70+
print "<table border>\n";
71+
print "<tr><th>Rank<th>Size&times;Age (GB-years)<th>File<th>Size (GB)<th>Last Access Time<th>Owner<th>Directory\n";
72+
$i = 1;
73+
while (@row = $sth->fetchrow_array) {
74+
$age_size = round($row[0]/$seconds_per_year);
75+
$size = round($row[2]/1.e9);
76+
$user = getpwuid($row[4]);
77+
if (! $user) {$user = "uid=$row[4]";}
78+
print "<tr><td>$i<td>$age_size<td>$row[1]<td>$size<td>$row[3]<td>$user<td>$row[5]\n";
79+
$i++;
80+
}
81+
print "</table>\n";
82+
83+
print $q->h2("Largest Directories"), "\nExcludes files in sub-directories\n";
84+
###################
85+
$sql = "select sum($file_table.size) as dirsize, dirname, $dir_table.uid from $dir_table, $file_table where dirId = $dir_table.id group by dirId order by dirsize desc limit 10;";
86+
make_query($dbh, \$sth);
87+
print "<table border>\n";
88+
print "<tr><th>Rank<th>Size (GB)<th>Directory<th>Owner\n";
89+
$i = 1;
90+
while (@row = $sth->fetchrow_array) {
91+
$size = round($row[0]/1.e9);
92+
$user = getpwuid($row[2]);
93+
if (! $user) {$user = "uid=$row[2]";}
94+
print "<tr><td>$i<td>$size<td>$row[1]<td>$user\n";
95+
$i++;
96+
}
97+
print "</table>\n";
98+
99+
print $q->h2("Directories with Greatest Average Age"),
100+
##########################################
101+
"\nExcludes files in sub-directories, size-weighted average age of all files in directory\n";
102+
103+
$sql = "select sum(($file_table.size*1.e-9)*(unix_timestamp(now()) - unix_timestamp(atime)))/sum($file_table.size)*1.e9 as aveage, dirname, $dir_table.uid from $dir_table, $file_table where $dir_table.id = dirId group by dirId order by aveage desc limit 10;";
104+
make_query($dbh, \$sth);
105+
print "<table border>\n";
106+
print "<tr><th>Rank<th>Age (years)<th>Directory<th>Owner\n";
107+
$i = 1;
108+
while (@row = $sth->fetchrow_array) {
109+
$age_size = round($row[0]/$seconds_per_year);
110+
$user = getpwuid($row[2]);
111+
if (! $user) {$user = "uid=$row[2]";}
112+
print "<tr><td>$i<td>$age_size<td>$row[1]<td>$user\n";
113+
$i++;
114+
}
115+
print "</table>\n";
116+
117+
print $q->h2("Directories with Greatest Size &times; Age"),
118+
##########################################
119+
"\nExcludes files in sub-directories, age &times; size summed over all files in directory\n";
120+
121+
$sql = "select sum(($file_table.size*1.e-9)*(unix_timestamp(now()) - unix_timestamp(atime))) as sumgby, dirname, $dir_table.uid from $dir_table, $file_table where $dir_table.id = dirId group by dirId order by sumgby desc limit 10;";
122+
make_query($dbh, \$sth);
123+
print "<table border>\n";
124+
print "<tr><th>Rank<th>Size&times;Age (GB-years)<th>Directory<th>Owner\n";
125+
$i = 1;
126+
while (@row = $sth->fetchrow_array) {
127+
$age_size = round($row[0]/$seconds_per_year);
128+
$user = getpwuid($row[2]);
129+
if (! $user) {$user = "uid=$row[2]";}
130+
print "<tr><td>$i<td>$age_size<td>$row[1]<td>$user\n";
131+
$i++;
132+
}
133+
print "</table>\n";
134+
135+
print $q->h2("Largest Total File Size by User"),
136+
###############################
137+
"\nSum of all files owned by user";
138+
139+
$sql = "select sum($file_table.size) as sumsize, $file_table.uid from $file_table group by $file_table.uid order by sumsize desc limit 10;";
140+
make_query($dbh, \$sth);
141+
print "<table border>\n";
142+
print "<tr><th>Rank<th>Total Size (GB)<th>User";
143+
$i = 1;
144+
while (@row = $sth->fetchrow_array) {
145+
$size = round($row[0]/1.e9);
146+
$user = getpwuid($row[1]);
147+
if (! $user) {$user = "uid=$row[1]";}
148+
print "<tr><td>$i<td>$size<td>$user\n";
149+
$i++;
150+
}
151+
print "</table>\n";
152+
153+
print $q->h2("Greatest Average File Age by User"),
154+
######################################
155+
"Size-weighted average age of all files owned by user\n";
156+
157+
$sql = "select sum(($file_table.size*1.e-9)*(unix_timestamp(now())-unix_timestamp(atime)))/sum($file_table.size)*1.e9 as aveage, $file_table.uid from $file_table group by $file_table.uid order by aveage desc limit 10;";
158+
make_query($dbh, \$sth);
159+
print "<table border>\n";
160+
print "<tr><th>Rank<th>Age (years)<th>User\n";
161+
$i = 1;
162+
while (@row = $sth->fetchrow_array) {
163+
$age_size = round($row[0]/$seconds_per_year);
164+
$user = getpwuid($row[1]);
165+
if (! $user) {$user = "uid=$row[1]";}
166+
print "<tr><td>$i<td>$age_size<td>$user\n";
167+
$i++;
168+
}
169+
print "</table>\n";
170+
171+
print $q->h2("Largest Total Size &times; Age by User"),
172+
######################################
173+
"\nAge &times; size summed over all files owned by user\n";
174+
175+
$sql = "select sum(($file_table.size*1.e-9)*(unix_timestamp(now())-unix_timestamp(atime))) as sumgby, $file_table.uid from $file_table group by $file_table.uid order by sumgby desc limit 10;";
176+
make_query($dbh, \$sth);
177+
print "<table border>\n";
178+
print "<tr><th>Rank<th>Size&times;Age (GB-years)<th>User\n";
179+
$i = 1;
180+
while (@row = $sth->fetchrow_array) {
181+
$age_size = round($row[0]/$seconds_per_year);
182+
$user = getpwuid($row[1]);
183+
if (! $user) {$user = "uid=$row[1]";}
184+
print "<tr><td>$i<td>$age_size<td>$user\n";
185+
$i++;
186+
}
187+
print "</table>\n";
188+
189+
print $q->end_html; # end the HTML
190+
191+
print "\n";
192+
193+
exit;
194+
#
195+
# make a query
196+
#
197+
sub make_query {
198+
199+
my($dbh, $sth_ref) = @_;
200+
$$sth_ref = $dbh->prepare($sql)
201+
or die "Can't prepare $sql: $dbh->errstr\n";
202+
203+
$rv = $$sth_ref->execute
204+
or die "Can't execute the query $sql\n error: $sth->errstr\n";
205+
206+
return 0;
207+
}
208+
sub round {
209+
my ($float) = @_;
210+
$rounded_float = sprintf("%.2f", $float);
211+
return $rounded_float;
212+
}
213+
#
214+
# end of file
215+
#

0 commit comments

Comments
 (0)