Here’s example Perl implementation of the Apache access.log filter for “telling human from computer [site visitors] apart”.
For the description please see main page.

#!/usr/bin/perl -w
#
# Extract human-like entries from httpd server log.
# Note:
# Some legitimate users may be filtered out,
# however, they probably are not interesting economically anyway,
# so not really required in the analysis.

my $logfile = defined $ARGV[0] ? $ARGV[0] : "";

# Pass 1: get the list of (spam) bots.
open(L, $logfile) or die "Logfile $logfile unaccessible (pass 1).\n";
my %bots = ();
my %humans = ();
my $bad_lines = 0;
my $MAX_BAD = 3;
my $verb = 0;
while () {
        /^(\S+) .+? (\S*?) HTTP\/\S*? (\d\d\d) / or do {
                ++$bad_lines;
                $verb and $bad_lines <= $MAX_BAD and
                        warn "Bad line: $_";
                next;
        };
        $host           = $1;
        $request        = $2;
        $status         = $3;
        if (like_human()) {
                $humans{$host} = 1;
                delete $bots{$host};
        }
        else {
                exists $humans{$host} or $bots{$host} = 1;
        }
}
close L;
$verb && printf STDERR " bots: %d, humans: %d, bad lines: %d\n",
        scalar keys %bots,
        scalar keys %humans,
        $bad_lines;

#Pass 2: extract human-like entries
open(L, $logfile) or die "Logfile $logfile unaccessible (pass 2).\n";
while () {
        /^(\S+) / or next;
        $host           = $1;
        print unless exists $bots{$host};
}
close L;

sub like_human
{
        return 1 if exists $humans{$host};
        return
                $request =~ /^\// &&
                $status eq "304" ||
                $request =~ /\.(?:js|css|jpe?g|png|gif|ico|pdf|mp3|avi)$/i
                # Is "htm" ok?..
                ?
                1 : 0;
}

← Rate!