#!/usr/bin/perl -w

# Logfile analysis script to run the "Analog" logfile analysis tool.

# Dean Pentcheff (dean2@biol.sc.edu).  Copyright 1997 and 2000 by the
# Trustees of the University of South Carolina.  This program is free
# software; you can redistribute it and/or modify it under the same
# terms as Perl itself.

# See the end of this file for changelog and documentation.

# $Id: runanalog,v 1.18 2001/06/11 20:11:41 dean2 Exp $

use strict;            # enforce strict interpretation of Perl syntax
use Getopt::Std;       # command-line option processing
use CGI qw(:standard :html3); # just for generating HTML text, no CGI
$CGI::Q = new CGI({}); #    and terminate its CGI processing loop

# full path to analog executable
my($analog) = '/usr/bin/analog';
# parent directory into which HTML files are dumped
my($htmldir) = '/home/apache/stats/';
# parent directory's URL (used only in the index page footer)
my($htmldirurl) = 'http://stats.bsa171.org/';
# analog's built-in image directory URL (needs terminal slash)
my($analogimageurl) = '/images/';
# filename of this-month-to-date file (within each server's output subdir)
my($current) = 'current.html';
# title for the top-level index page itself (not the stats pages themselves)
my($title) = 'Usage Statistics for Web Sites Hosted by BIERSTADT.BSA171.ORG';
# filename of the top-level HTML index file (within $htmldir)
my($index) = 'index.html';
# html colors for top-level HTML index file
my($pagecol) = 'white';    # index page background color
my($oddcol)  = '#ffcc99';  # index page odd data columns color
my($evencol) = '#ffcc66';  # index page even data columns color
my($headcol) = '#ff9966';  # index page header cell color
my($nullcol) = '#cccccc';  # index page empty cell color
my($ts) = `date`; chomp($ts);
my($footer)  = <<"EndOfFooter";  # index page ending boilerplate
<hr>
<center><font size="-2">
Logfile analysis:
<a href="http://www.statslab.cam.ac.uk/~sret1/analog/">Analog</a>
by Stephen Turner.<br>
Original driver script: <a href="http://www.biol.sc.edu/usage/runanalog">runanalog</a>
(<a href=//www.biol.sc.edu/usage/runanalog.html>man page</a>)
by Dean Pentcheff (<a href="mailto:dean2\@biol.sc.edu">dean2\@biol.sc.edu</a>),
Biological Sciences, University of South Carolina<br>
Modified driver script: <a href="runanalog">runanalog</a> by Ryan Kirkpatrick (<a href="mailto:webmaster\@rkirkpat.net">webmaster\@rkirkpat.net</a>)<br>
Last updated: $ts
</font></center>
EndOfFooter

my(%defaults) =
    (
     ### Probably invariant across servers (but could - override in %servers)
     # arguments for the monthly analog reports
     analogargs => [qw(-m +W +D +d -H +h -4 -5 +Sr-100 +Z +o +r +i +t +z),
		    qw(+P +E +I +f +s +N +n +k +K +B +b +p -v +u +J +c),
		    "-CREPORTORDER xmWDdHh45oZSkKfsNnBbpvirPEIuJztc1Qw67lLRMjYy", # phew
		    "-CWARNINGS -R",          # suppress null-report warnings
		    "-CLASTSEVEN OFF",        # no lastweek bits
		    "-CGOTOS FEW",            # hoplinks only at top & end
		    "-CHOSTURL none",         # no hostlink in title
		    "-CSUBDIR /*/*/*",        # hierarchical directory report
		    "-CDIRCOLS NRb",          # so give us an index column
		    "-CSUBDOMAIN *.*",        # hierarchical domain report
		    "-CDOMCOLS NRb",          # so give us an index column
		    "-CPAGEINCLUDE *.cgi," .  # supplement defined "pages"
		    "*.php,*.php3,*.pdf,*.PDF,*.doc,*.DOC,*.txt,*.TXT",
                    "-CDNS WRITE",            # Enable DNS lookups.
                    "-CDNSFILE /var/log/apache/dnsfile.txt",
                    "-CDNSLOCKFILE /var/log/apache/dnslock",
		   ],
     # path to the www server's log directory
     logdir => '/var/log/apache/',
     ### Per-server information - override in %servers on a per-server basis
     # display name of the server
     hosttitle => '',
     # real name of the server for making links on stats pages
     hostname => '',
     # rootname for the logfiles within the logdir directory
     logname => '',
     # directory name for HTML output within the htmldir usage directory
     hostdir => '',
     # placeholder for additional analog args added in the %servers hash
     analogextra => [],
     # link target for use in the HTML index page's host column title
     tablelink => '',
    );

my(%servers,@serverOrder);
@serverOrder = ( "www", "ssl", "ssls", "db", "mail");
%servers = (
            mail   => {hosttitle    => 'mail.b.o',
                       hostname     => 'mail.bsa171.org',
                       logname      => 'mail-access.log',
                       hostdir      => 'mail',
                       tablelink    => 'http://mail.bsa171.org/',
                      },
            ssl   => {hosttitle    => 'ssl.b.o',
                      hostname     => 'ssl.bsa171.org',
                      logname      => 'ssl-http-access.log',
                      hostdir      => 'ssl',
                      tablelink    => 'http://ssl.bsa171.org/',
                     },
            ssls  => {hosttitle    => 'ssl.b.o<BR>(SSL)',
                      hostname     => 'ssl.bsa171.org',
                      logname      => 'ssl-https-access.log',
                      hostdir      => 'ssls',
                      tablelink    => 'https://ssl.bsa171.org/',
                     },
            www   => {hosttitle    => 'www.bsa171.org',
                      hostname     => 'www.bsa171.org',
                      logname      => 'www-access.log',
                      hostdir      => 'www',
                      tablelink    => 'http://www.bsa171.org/',
                     },
            db => {hosttitle => 'db.b.o',
                   hostname  => 'db.bsa171.org',
                   logname   => 'db-access.log',
                   hostdir   => 'db',
                   tablelink => 'http://db.bsa171.org/',
                  }
   );


sub help {
    print STDERR <<EOHelp;
Command-line options:
    -f         Force 1st-of-the-month processing: summarize last month
                 (except on 1st of month, when it quells first-day processing)
    -x         Just make the index file, don't read logfiles
    -o         Overwrite existing date-named files (current always overwrites)
    -w server  Pick webserver(s) using regular expressions on their tags
    -v         Run verbosely
    -n         Fake the run, you'll want to use -v as well
    -d         Turn on the debug flag - see code for effects (also forces -v)
    -h         Show this short help
    -s YYYYMM  Force "current date" to YYYYMM (Jan=1, Dec=12)
EOHelp
    exit 0;
}

# Get any commandline options
my(%options);
getopts('fxnovdhw:s:', \%options);
my($forcefirst)  = exists($options{'f'});
my($indexonly)   = exists($options{'x'});
my($fake)        = exists($options{'n'});
my($overwrite)   = exists($options{'o'});
my($verbose)     = exists($options{'v'}) || exists($options{'d'});
my($debug)       = exists($options{'d'});
help()          if exists($options{'h'});
my($serverre)    = exists($options{'w'})  ?  $options{'w'}  :  '.*';
my($setdate)     = exists($options{'s'})  ?  $options{'s'}  :  '';
print "Options: " .
    "forcefirst=$forcefirst " .
    "indexonly=$indexonly " .
    "overwrite=$overwrite " .
    "fake=$fake " .
    "verbose=$verbose " .
    "debug=$debug " .
    "setdate=$setdate " .
    "\n"  if ($verbose);


# Figure out our timing information
my($mday, $mon, $year) = (localtime)[3,4,5];
$year += 1900; # give us a 4-digit year
$mon  += 1;    # give us a month where Jan==1
if ($setdate) {
    ($year, $mon) = $setdate =~ /^(\d{4})(\d{2})/;
    die("Bad set date format '$setdate' (should be YYYYMM)\n")
	unless ($year  &&  $mon);
    die("Specified year must be > 1970")
	unless ($year > 1970);
    die("Specified month must be between 1 and 12 inclusive")
	unless ($mon >= 1  &&  $mon <= 12);
}
# do we actually want to summarize last month?
my($firstday) =
    (($mday == 1 && ! $forcefirst) ||  ($mday != 1  &&  $forcefirst));
if ($firstday) {
    if (--$mon < 1) {
	--$year;  # we were in Jan, so set for Dec of last year
	$mon = 12;
    }
}
my($yy) = substr($year, 2, 2); # get a two-digit year for analog's use
printf STDERR "Analysis month is: %02d, %4d\n", $mon, $year if ($verbose);

# shortcut here if we're just (re)making the index file
if ($indexonly) {
    print STDERR "just making indexfile $htmldir/$index\n" if ($verbose);
    make_index($title, $htmldir, $index, $current, \@serverOrder, \%servers);
    exit 0;
}


# now step through each of the desired servers
my @order;
foreach (@serverOrder) {
    /$serverre/ and push(@order,$_);
}
@serverOrder = @order;
print STDERR "Remaining servers: ", join(" ", keys %servers), "\n" if $verbose;
foreach (@serverOrder) {
    # merge the per-server config with the default/generic config
    my($servertag) = $_;
    my(%cnf) = %defaults;
    for (keys %{$servers{$servertag}}) {
	if (exists $cnf{$_}) {
	    $cnf{$_} = $servers{$servertag}->{$_};
	} else {
	    print STDERR "Warning: server '$servertag', bogus config: '$_'\n";
	}
    }

    # get list of access_log files, in lexicographic == time-ascending order
    opendir(DIR, $cnf{logdir})  ||  die("Failed to open $cnf{logdir}: $!");
    my(@logfiles) = map { "$cnf{logdir}/$_" }
	            sort {$b cmp $a}
	            grep(/^$cnf{logname}(|\.\d+)$/, readdir(DIR));
    closedir(DIR);
    @logfiles = grep( ! /\.[234]/, @logfiles) if $debug;
    print STDERR "Log files are:\n\t", join("\n\t", @logfiles), "\n"
	if ($verbose);

    # make sure we've got an output directory, or make one
    die("Missing output directory!: $htmldir") unless -d $htmldir;
    unless (-d "$htmldir/$cnf{hostdir}") {
	mkdir "$htmldir/$cnf{hostdir}", 0755 or
	    die("Failed mkdir '$htmldir/$cnf{hostdir}': $!");
    }

    # blank out the "current" file if we're on the first of the month
    empty_current("$htmldir/$cnf{hostdir}/$current") if ($firstday);

    # get our analog run timing and outfile args ready
    my($from, $to, $out);
    $from = sprintf("+F%02d%02d01", $yy, $mon);
    if ($setdate  ||  $firstday) {
	$to  = sprintf("+T%02d%02d31", $yy, $mon); # 31 is ok for all months
	$out = sprintf("$htmldir/$cnf{hostdir}/%4d%02d.html", $year, $mon);
	if (-e $out  &&  ! $overwrite) {
	    print STDERR "Will not overwrite existing '$out'\n";
	    next;
	}
    } else {
	$to  = sprintf("+T%02d%02d-01", $yy, $mon); # through yesterday
	$out =         "$htmldir/$cnf{hostdir}/$current";
    }

    # get an HTML-cleaned version of the hostname for title in page
    my($title) = $cnf{hosttitle};
    $title =~ s/<.*?>/ /g;

    # do the damned run already, fer crissake
    my($rc)   = 0;    # analog run return code
    my(@cmd) = ($analog, @{$cnf{analogargs}}, $from, $to,
		"-COUTFILE $out",                  # output file
		"-CIMAGEDIR $analogimageurl",      # URL for analog's images
		"-CHOSTNAME \"$title\"",    # for title displays
		"-CBASEURL http://$cnf{hostname}", # for internal links
		@{$cnf{analogextra}},              # additional analog args
		map { "-CLOGFILE $_" } @logfiles
		);
    printf STDERR "@cmd\n" if ($verbose);
    unless ($fake) {
	$rc = 0xffff & system(@cmd);
	unless ($rc == 0) {
	    $rc = sprintf("%#04x", $rc);
	    warn("$analog returned code $rc: $!");
	}
    }
}

# find all historic and currentfiles and (re)create toplevel HTML page
make_index($title, $htmldir, $index, $current, \@serverOrder, \%servers);

print STDERR "Done.\n" if ($verbose);
exit 0;





# Generate an empty "none here yet" version of the HTML currentfile
sub empty_current {
    my($file) = @_;

    print STDERR "emptying 'current' file '$file'\n" if ($verbose);

    return if $fake;

    open(OUT, "> $file") || die("Can't open current-stats file '$file': $!\n");
    print OUT start_html(-title => 'Month-to-Date Log Statistics',
			 -bgcolor => $pagecol),
	h1('Month-to-Date Log Statistics'),
	h2('This Month to Date'),
	p(),
        i('No month-to-date statistics yet for this month. ' .
          'See historical listings for statistics up to now.'),
        end_html();
    close OUT;
}



# Generate the HTML index page linking to historical and currentfiles
sub make_index {
    my($title, $dir, $index, $current, $order, $list) = @_;
    my(@months) = ('',   # months are based on Jan=1, so have an empty 0th
		   qw(January February March April May June July
		      August September October November December));
    my(%files);

    print STDERR "making indexfile $htmldir/$index\n" if ($verbose);

    # mine all files, note oldest/newest, and pluck some stats
    my($oldest) = '999999';
    my($newest) = '000000';
    foreach (@$order) {
	my($servertag) = $_;
	my($hostdir) = $list->{$servertag}->{hostdir};
	opendir(DIR, "$dir/$hostdir") or
	    die("Failed to open directory '$dir/$hostdir': $!");
	my(@histfiles) = sort {$b cmp $a}
	    grep(/^(\d{6}\.html|$current)$/, readdir(DIR));
	closedir(DIR);
	print STDERR "in $dir/$hostdir:\n\t", join("\n\t", @histfiles), "\n"
	    if $verbose;
	# assemble a hash, keyed by yyyymm or 'current', with stats line
	for (@histfiles) {
	    my($file) = "$dir/$hostdir/$_";
	    my($filetag);
	    if (/^(\d{6})/) {
		$filetag = $1;
		$oldest = $filetag  if $filetag < $oldest;
		$newest = $filetag  if $filetag > $newest;
	    } elsif (/^$current/) {
		$filetag = $current;
	    } else {
		die("Program error - should be either dated or current");
	    }
	    # grub
	    my($reqs)  = '';
	    my($bytes) = '';
	    open(FILE, $file) or die("Failed to open '$file' for reading: $!");
	    while (<FILE>) {
		$reqs  = $2 if (/(Total s|S)uccessful requests:[^\d]*(.*)/);
		$bytes = $2 if (/(Total d|D)ata transferred:[^\d]*(.*)/);
		last if ($reqs  &&  $bytes);
	    }
	    close FILE;
	    foreach ($reqs, $bytes) {
		next unless $_;
		my($mult) = 1;
		$mult = 2**10 if (/kbyte/i || /kilobytes/i);
		$mult = 2**20 if (/mbyte/i || /megabytes/i);
		$mult = 2**30 if (/gbyte/i || /gigabytes/i);
		s/[^0-9\.]//g;
		$_ *= $mult;
	    }
	    $bytes = int($bytes / 1048576 + 0.5) if ($bytes); # force to MB
	    1 while $bytes =~ s/^(-?\d+)(\d{3})/$1,$2/; # commify
	    1 while $reqs  =~ s/^(-?\d+)(\d{3})/$1,$2/; # commify
	    if ($reqs) {  # allow 0 $bytes, since < 0.5MB goes to 0
		$files{$servertag}{$filetag} = [$reqs, $bytes];
	    } else {
		# suppress indexing this file if there's nothing in it
		delete $files{$servertag}{$filetag};
	    }
	}
    }
    print STDERR "Oldest = $oldest   Newest = $newest\n" if $verbose;

    return if $fake;

    # massage things into an output table
    open(IDX, "> $dir/$index") || die("Failed to open $dir/$index: $!");
    print IDX start_html(-title => $title,
			 -bgcolor => $pagecol,
			 -meta => {"robots", "noindex,nofollow",
				  },
			),
	qq|<center><h1>$title</h1>|,
	qq|<table cellspacing=1 cellpadding=2 border=0>\n|,
	qq|<tr><td bgcolor="$headcol">&nbsp;</td>|;
    foreach (@$order) {
	print IDX qq|<th colspan=2 bgcolor="$headcol">|;
	print IDX qq|<a href="$list->{$_}->{tablelink}">|
	    if exists $list->{$_}->{tablelink};
	print IDX $list->{$_}->{hosttitle};
	print IDX qq|</a>|
	    if exists $list->{$_}->{tablelink};
	print IDX qq|</th>|;
    }
    print IDX qq|</tr><tr><td bgcolor="$headcol">&nbsp;</td>\n|;
    foreach (@$order) {
	print IDX qq|<td bgcolor="$oddcol" align=center><font size=-1>|,
	    qq|Requests</font></td>|,
	    qq|<td bgcolor="$evencol" align="center" valign="bottom">|,
	    qq|<font size=-2>MB</font></td>|;
    }
    print IDX "</tr>\n";

    # the 'current' entries, first data row, suppressed on first of the month
    my($anycurrents) = 0;
    # first, are there any current entries?
    foreach (@$order) {
	++$anycurrents if exists $files{$_}->{$current};
    }
    # if so, punch out a line for them
    if ($anycurrents) {
	print IDX qq|<tr><td bgcolor="$headcol">Month-to-Date</td>|;
	foreach (@$order) {
	    if (exists $files{$_}->{$current}) {
		my($reqs, $bytes) = @{$files{$_}->{$current}};
		print IDX
		    qq|<td align=right bgcolor="$oddcol">|,
		    qq|<a href="$list->{$_}->{hostdir}/$current">|,
		    qq|$reqs</a></td>|,
		    qq|<td align=right valign="bottom" bgcolor="$evencol">|,
		    qq|<font size=-1>$bytes</font></td>|;
	    } else {
		print IDX
		    qq|<td bgcolor="$nullcol">&nbsp;</td>|,
		    qq|<td bgcolor="$nullcol" valign="bottom">|,
		    qq|<font size=-1>&nbsp;</font></td>|;
	    }
	}
	print IDX "</tr>\n";
    }

    # now fill in all available historical months
    my($newestyear, $newestmonth) = $newest =~ /^(\d{4})(\d{2})/;
    my($oldestyear, $oldestmonth) = $oldest =~ /^(\d{4})(\d{2})/;
    my($year) = $newestyear;
    my($month) = $newestmonth;
    while ($year > $oldestyear  ||
	   ($year == $oldestyear  &&  $month >= $oldestmonth) ) {
	my($yyyymm) = sprintf "%04d%02d", $year, $month;
	print IDX qq|<tr><td bgcolor="$headcol">|,
	    qq|$year&nbsp;$months[$month]</td>|;
	foreach (@$order) {
	    if (exists $files{$_}->{$yyyymm}) {
		my($reqs, $bytes) = @{$files{$_}->{$yyyymm}};
		$reqs  = '&nbsp;' unless $reqs  ne '';
		$bytes = '&nbsp;' unless $bytes ne '';
		print IDX
		    qq|<td align=right bgcolor="$oddcol"><a href=|,
                    qq|"$list->{$_}->{hostdir}/$yyyymm.html\">|,
		    qq|$reqs</a></td>|,
		    qq|<td align="right" valign="bottom" bgcolor="$evencol">|,
		    qq|<font size=-1>$bytes</font></td>|;
	    } else {
		print IDX
		    qq|<td bgcolor="$nullcol">&nbsp;</td>|,
		    qq|<td bgcolor="$nullcol" valign="bottom">|,
		    qq|<font size=-1>&nbsp;</font></td>|;
	    }
	}
	print IDX "</tr>\n";
	if (--$month < 1) {
	    $month = 12;
	    --$year;
	}
    }
   print IDX "</table></center>", $footer, end_html;
    close IDX;
}

__END__

# $Log: runanalog,v $
# Revision 1.18  2001/06/11 20:11:41  dean2
# Added check to put in a &nbsp; in the case of empty requests
# or bytes in the top-level index page.  Needed to add the old
# tbone logfiles, which lack bytecounts.
#
# Revision 1.17  2001/02/02 05:44:15  dean2
# Did some aliasing and argsexcluding on the www-s server.
# Added a space when purging HTML from host titles.
# Fixed up the "-h" help strings.
#
# Revision 1.16  2001/02/02 02:18:22  dean2
# Fixed a missing ">" in the current-file link in the toplevel index file.
# Added function to the "-f" switch so that it turns _off_
# first-of-the-month processing on the first of the month.
# Updated documentation to fit.
#
# Revision 1.15  2001/01/26 17:39:25  dean2
# Changed column titles slightly.
#
# Revision 1.14  2001/01/26 04:03:21  dean2
# Added the USC-only and nonUSC-only breakdowns for www.
# Shuffled host order yet again.
#
# Revision 1.13  2001/01/26 02:15:37  dean2
# Shuffled the server order.
#
# Revision 1.12  2001/01/26 02:09:52  dean2
# Moved gator from remote to local logfile location.
# Left hughes there (though it is now dead) to document lack of use to date.
#
# Revision 1.11  2001/01/23 09:27:44  dean2
# Redid the output display to better visually separate
# hit counts from byte counts in the index table.
#
# Revision 1.10  2001/01/23 08:46:13  dean2
# Added the "tablelink" item to the server hash, so that we could put up
# links to the Netcraft server uptime graphs.
# Added marine.geol.sc.edu to the lineup.
#
# Revision 1.9  2001/01/15 22:51:45  dean2
# Added darwin and darwin-proxy to the lineup.
#
# Revision 1.8  2001/01/11 17:57:42  dean2
# Moved zebra's log location from the "remote" directory to the normal one.
#
# Revision 1.7  2000/08/10 17:16:13  dean2
# Replaced space between year and month with a non-break space.
#
# Revision 1.6  2000/08/07 17:56:29  dean2
# Minor doc changes.
#
# Revision 1.5  2000/08/06 03:46:39  dean2
# Made some reports hierarchical.
# Tweaked output format and report order.
#
# Revision 1.4  2000/08/05 03:04:04  dean2
# Changed some file locations.
# Modified arglist to use -CLOGFILE instead of a list of filenames;
# analog seemed unhappy with a long comma-separated list.
#
# Revision 1.3  2000/08/04 23:05:32  dean2
# Patched docs, tweaked HTML.
#
# Revision 1.2  2000/08/04 21:46:24  dean2
# First tentative production version.
# Wrote pod documentation.
#
# Revision 1.1  2000/08/04 09:02:31  dean2
# Initial revision
#


# startopod

=pod

=head1 NAME

runanalog - wrapper to run the Analog webfile analysis program

=head1 SYNOPSIS

runanalog [arguments]

=head1 DESCRIPTION

The Analog web log analyzer needs to be run on a periodic basis with
specified arguments to generate any sort of sequential log files.
This program, run daily by L<cron(8)> or another suitable scheduler, will
call Analog appropriately.  This program does I<not> rotate log files
or schedule its own invocation.

Each server gets its own subdirectory for Analog output files, and
each month's summary file is named using the date (I<YYYYMM.html>).
In the parent directory is an index page consisting of an HTML table
with one row per month and a double column for each server analyzed.
Each row for each server shows the total successful accesses and MB
transferred (harvested from the Analog analysis files).  Each of those
numbers is a link to the full Analog analysis file for that server for
that month.

It assumes the following:

=over

A server's logfiles are in a single directory

A month's worth of logfiles can remain in that directory
through the first of the subsequent month.

The logfiles are small enough that it's OK to reread them all
each day when this program is run.  (Analog is very fast, so this is
probably less of an issue than it sounds.)

Multiple logfile generations accumulate with the following naming
pattern in increasing age (the logfile base name is configurable):
F<logfile> F<logfile.1> F<logfile.2> ...

=back


=head2 Commandline arguments

=over

=item -x

Just make the top-level index file based on existing Analog output
files.  Don't read logfiles or run Analog itself.

=item -o

Force overwriting of existing Analog output files.  By default, if a
monthly summary already exists, it will not be overwritten (and a
warning message is sent to STDERR).  I<NOTE> however that the
"current" (this-month-to-date) file can always be overwritten.

=item -w servername

Process only F<servername>'s log entries.  F<servername> can be any
Perl regular expression, and is matched against the server name keys
in the I<%servers> hash (see configuration details).  Hence it can
select more than one server's logs for processing.  (Note that if you
include special characters like "|" or "*" in the regular expression,
you will probably have to enclose this argument in single quotes to
protect it from expansion by your commandline shell.)

=item -s YYYYMM

Force our idea of the "current date" to YYYYMM.  It must be a
four-digit year followed by a two-digit month (Jan=01, Dec=12).

=item -f

On any day execept the first of the month, force "first of the month"
processing: produce a complete summary of I<last> month's statistics,
and generate an empty "current statistics" file (since we only ever
summarize through the preceeding midnight, there are no month-to-date
numbers for the first day of each month).

If it really is the first day of the month, then B<-f> has the
opposite effect: forces the program to ignore the fact that we are
running it on the first of the month.  Trying to process older data
using the B<-s> flag on the first of the month will cause
off-by-one-month results if you don't supply the B<-f> flag as well.

=item -v

Run verbosely, telling us what's happening.

=item -n

Fake the run, avoid actually writing anything or running Analog
itself.  (Probably you want B<-v> as well.)

=item -h

Print a little command-line argument summary and quit.

=item -d

Turn on a debugging flag, which may do something the programmer was
interested the last time he tweaked the code -- check the code for
what this does.

=back


=head2 Configuration

B<Runanalog> is intended to be run nightly by L<cron(8)> or a similar
scheduling program.  In normal use it needs no commandline arguments.
Check your system docs to see how to run a program periodically using
L<cron(8)> (or similar).  It's probably best to run this early in the
morning so that the 'current' summary covers through yesterday.

Configuration is done by setting some variables in the program
text.  There are three main sections.  (I know this could be done
using an external config file, but I really couldn't be bothered.)

Starting at the beginning are some global scalars.  Set them
appropriately to point to your program locations, directories, etc.
Note that I<$htmldir> is a parent directory for the analysis
output: it will contain an overall index file and one subdirectory
for each server's file analyzed.  Comments preceeding each
describe their purpose.

Next is the I<%defaults> hash.  These values are set as the default
for all servers, and then server-specific values (where needed)
are specified (see next paragraph).  The main things to check here
are the Analog argument string (I<analogargs>) and the full path
to the server log files (I<logdir>).  Leave the per-server entries
empty.

Last is the I<%servers> hash.  This has one clause for each server's
files you wish to analyze.  The columns in the overall index file will
be in the same order as the items initializing this hash.  Here's one
that will run analysis on two servers:

 tie(%servers, 'Tie::IxHash',
    www     => {hosttitle    => 'www',
		hostname     => 'www.biol.sc.edu',
		logname      => 'www.access_log',
		hostdir      => 'www',
	       },
    mail    => {hosttitle    => 'mail host',
		hostname     => 'mail.biol.sc.edu',
		logname      => 'mail.access_log',
		hostdir      => 'mail',
		analogextra  => ['-CALLGRAPH R', '-CRAWBYTES ON'],
		tablelink    => 'http://www.netcraft.com/whats?host=mail.biol.sc.edu',
	       },
 };

The first server, identified by the key C<www>, is probably the most
simple setup you'll have.  The server's name for page titles will be
I<www>, the hostname for use in web links within Analog's reports will
be C<www.biol.sc.edu>, the logfiles all begin with F<www.access_log>,
and are found in the I<logdir> directory noted in the I<%defaults>
hash (if that were not true, a I<logdir> entry could be put into
I<www>'s clause here).  This server's Analog output files will be put
in a subdirectory named "www" (the I<hostdir> value) within the
I<$htmldir> directory configured in the scalar variables above.

The next entry is similar, but adds the I<analogextra> item to tack on
a few additional Analog command-line variables.  Such variables should
be in the form of an anonymous array, one array item per argument.  It
also adds the I<tablelink> item which can hold a link target for the
host's column header in the indexing HTML page that will be generated.

=head1 EXAMPLES

=over

=item Daily run by cron(8) in the early morning:

 runanalog

No arguments at all cause it to parse all logfiles for all servers in
the I<%servers> hash, summarizing the stats for this month to date
into the 'current' file in each server's output directory, and
recreate the overall index file table in I<$htmldir>.  If and only if
it is the first day of the month, the preceeding month's statistics
will be generated into a file (named YYYYMM.html) in each server's
output directory and their 'current' files will be effectively emptied
(since we only ever summarize through the preceeding midnight, there
are no month-to-date numbers for the first day of each month).

=item Initial run with existing logfiles covering several months:

 mkdir /usr/local/apache/htdocs/usage
 runanalog -s 199912
 runanalog -s 200001
 runanalog -s 200002
 runanalog

You've just compiled Analog, grabbed this program, configured it, and
you'd like to catch up on your log file summaries.  This example
assumes that the date today is in the middle of March, 2000 and you've
got logfiles running back through December, 1999.  First create your
output directory (a subdirectory for each server will be automatically
created in that directory).  Then do a run for each full month
preceeding the current month.  Finally, do a run to summarize the
current month to date.  (Note that if you happened to be doing this on
the first day of a month, you'd want to use the B<-f> flag along with
the B<-s> flags to keep the program from doing "first of the month"
processing.)

=item Play with index page colors or formatting:

 runanalog -x

This will just read the existing Analog output files and regenerate
the I<$index> page, without reading any web log files or running Analog.

=item Fix recent summaries, or you don't run runanalog every day:

 runanalog -f
 runanalog

Something weird happened last month or early this month, so you'd like
to update last month's summaries and bring things up to date.  The
first line pretends to be running on the first of the month to
summarize last month (into a date-named file), then the second line
summarizes the current month to date.  If you're not running
I<runanalog> every day, then the first time you run it each month
you'll want to do both the B<-f> run to get the previous month neatly
packaged off, then the regular argumentless run to summarize the
current month to date.  (Note that if it just happens to be the first
of the month when you run this, then you won't need the B<-f> flag.)

=item Limit the run to only two servers:

 runanalog -w 'www|stkctr' -s 200002
 runanalog -w 'www|stkctr'
 runanalog -x

The first line will summarize February 2000 logs for servers matching
either "www" or "stkctr" as their I<%servers> hash key, and regenerate
the top-level index file including only those two servers.  The second
line will do a month-to-date run for just those servers, and again
regenerate the index file.  The third line would regenerate the
top-level index file for all the servers after you finished selective
runs.  This is an efficient sequence to introduce a new server or
servers to the summary system without reanalyzing the older logs for
the existing servers.  (Note that if you happened to be doing this on
the first day of a month, you'd want to use the B<-f> flag along with
the B<-s> flag to keep the program from doing "first of the month"
processing.)

=back

=head1 BUGS

No doubt.

=head1 FILES

(Referred to by their scalar variable names in the program.)

=over

=item $analog

Path to the Analog executable program.

=item $htmldir

Path for the parent directory into which all the Analog analyses
and the overall index file are placed.  Each server's analyses
get a subdirectory within this directory.

=item $analogimageurl

Analog comes with a directory of small images it uses to build up
graphs (and its icon).  This should be the URL of the directory
containing those graphics for use in Analog's output pages.

=item $current

The filename used for the 'current' or month-to-date summary in each
server's directory, initially set to F<current.html>.  It's probably
best I<not> to use F<index.html> for this, or people will be prevented
from browsing the directory for prior analyses.

=item $index

The filename used for the overall summary index file in the
I<$htmldir> directory.  Initially set to F<index.html>.

=back

=head1 AUTHOR

Dean Pentcheff (dean2@biol.sc.edu).  Copyright 1997 and 2000 by the
Trustees of the University of South Carolina.  This program is free
software; you can redistribute it and/or modify it under the same
terms as Perl itself.

=head1 SEE ALSO

Analog logfile analysis program from
http://www.statslab.cam.ac.uk/~sret1/analog/

Perl module L<Tie::IxHash> for hashes with ordered keys.

Perl module L<CGI> for HTML formatting.

=cut

# endopod

############ end of file ############