#!/usr/bin/env perl

use warnings;
use strict;

use DBI;
use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; # Socket for mysql
use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
use Nebulous::Client;


my ($db_host, $db_name, $db_user, $db_pw); # Database details
my ($label);                               # Label
my ($group);                               # Data group
my ($stage);                               # Stage of interest
my ($fault);                               # Fault code of interest
my ($limit);                               # Limit for query

GetOptions(
           'dbhost=s' => \$db_host, # Database host name
           'dbname=s' => \$db_name, # Database name
           'dbuser=s' => \$db_user, # Database user
           'dbpass=s' => \$db_pw, # Database p/w
           'stage=s' => \$stage, # Stage
           'label=s@' => \$label, # Label(s)
           'data_group=s@' => \$group, # Data group(s)
           'fault=s' => \$fault, # Fault code
           'limit=s' => \$limit, # Limit to query
           ) or
die "Unable to parse arguments.\n";
die "Unknown option: @ARGV\n" if @ARGV;
die "Required options: --dbhost --dbname --dbuser --dbpass --stage\n"
    unless defined $db_host
    and defined $db_name
    and defined $db_user
    and defined $db_pw
    and defined $stage;

# Database connection
my $db = DBI->connect( "DBI:mysql:database=$db_name;host=$db_host;mysql_socket=" . DB_SOCKET(),
                       $db_user,
                       $db_pw,
                       { RaiseError => 1, AutoCommit => 1 }
                       ) or die "Unable to connect to database: $DBI::errstr";

# Nebulous client
my $neb = Nebulous::Client->new( proxy => $ENV{NEB_SERVER} ) or die "Unable to connect to Nebulous: $!\n";

my $sql;                        # Query to run
my $fault_table;                # Table with fault
my $label_table;                # Table with label
if ($stage eq "diff") {
    $sql = "SELECT diff_id, skycell_id, hostname, path_base FROM diffRun JOIN diffSkyfile USING(diff_id) WHERE fault != 0 AND state = 'new'";
    $label_table = "diffRun";
    $fault_table = "diffSkyfile";
} elsif ($stage eq "stack") {
    $sql = "SELECT stack_id, 'stack', hostname, path_base FROM stackRun JOIN stackSumSkyfile USING(stack_id) WHERE fault != 0 AND state = 'new'";
    $label_table = "stackRun";
    $fault_table = "stackSumSkyfile";
} elsif ($stage eq "warp") {
    $sql = "SELECT warp_id, skycell_id, hostname, path_base FROM warpRun JOIN warpSkyfile USING(warp_id) WHERE fault != 0 AND state = 'new'";
    $label_table = "warpRun";
    $fault_table = "warpSkyfile";
} elsif ($stage eq "magic") {
    $sql = "SELECT magic_id, node, 'unknown' AS hostname, CONCAT(workdir, '/', exp_id, '/', exp_id, '.mgc.', magic_id, '.', node) FROM magicRun JOIN magicNodeResult USING(magic_id) WHERE magicNodeResult.fault != 0 AND state = 'new'";
    $label_table = "magicRun";
    $fault_table = "magicNodeResult";
} elsif ($stage eq "chip") {
    $sql = "SELECT chip_id, class_id, hostname, CONCAT(path_base, '.', class_id) FROM chipRun JOIN chipProcessedImfile USING(chip_id) WHERE fault != 0 AND state = 'new'";
    $label_table = "chipRun";
    $fault_table = "chipProcessedImfile";
} elsif ($stage eq "camera") {
    $sql = "SELECT cam_id, 'cam', hostname, path_base FROM camRun JOIN camProcessedExp USING(cam_id) WHERE fault != 0 AND state = 'new'";
    $label_table = "camRun";
    $fault_table = "camProcessedExp";
} elsif ($stage eq "destreak") {
    $sql = "SELECT magic_ds_id, component, 'unknown', CONCAT_WS('.', CONCAT(outroot, '/', exp_id), 'mds', magic_ds_id, stage_id, component) FROM magicDSRun JOIN magicDSFile USING(magic_ds_id) JOIN magicRun USING(magic_id) WHERE magicDSFile.fault != 0 AND magicDSRun.state = 'new'";
    $label_table = "magicDSRun";
    $fault_table = "magicDSFile";
} elsif ($stage eq "publish") {
    $sql = "SELECT pub_id, client_id, hostname, path_base FROM publishRun JOIN publishDone USING(pub_id) WHERE fault != 0 AND state = 'new'";
    $label_table = "publishRun";
    $fault_table = "publishDone";
} elsif ($stage eq "diffphot") {
    $sql = "SELECT diff_phot_id, skycell_id, hostname, path_base FROM diffPhotRun JOIN diffPhotSkyfile USING(diff_phot_id) WHERE fault != 0 AND state = 'new'";
    $label_table = "diffPhotRun";
    $fault_table = "diffPhotSkycell";
} elsif ($stage eq "chip_bg") {
    $sql = "SELECT chip_bg_id, class_id, hostname, path_base FROM chipBackgroundRun JOIN chipBackgroundImfile USING(chip_bg_id) WHERE fault != 0 AND state = 'new'";
    $label_table = "chipBackgroundRun";
    $fault_table = "chipBackgroundImfile";
} elsif ($stage eq "warp_bg") {
    $sql = "SELECT warp_bg_id, skycell_id, hostname, path_base FROM warpBackgroundRun JOIN warpBackgroundSkyfile USING(warp_bg_id) WHERE fault != 0 AND state = 'new'";
    $label_table = "warpBackgroundRun";
    $fault_table = "warpBackgroundSkyfile";
} else {
    die "Unsupported stage: $stage\n";
}

my $where = "";                 # WHERE restriction
$where .= " AND ($label_table.label LIKE '" . join("' OR $label_table.label LIKE '", @$label) . "')" if defined $label;
$where .= " AND ($label_table.data_group LIKE '" . join("' OR $label_table.data_group LIKE '", @$group) . "')" if defined $group;
$where .= " AND $fault_table.fault = $fault" if defined $fault;

$sql .= $where if length $where > 0;
$sql .= " ORDER BY rand() LIMIT $limit" if defined $limit;

my $rows = $db->selectall_arrayref( $sql ) or die "Unable to execute SQL: $DBI::errstr";
$db->disconnect;

my %assertion;
my %cannot_find_file;
my %failed_real_name;
my %failed_close;
my %unable_access;
my %statistics_file;
my %transparencies;
my %couldnt_open_file;
my %failed_read;
my %no_instances_available;
my %neb_entry;
my %variance;
my %error_reading;
my %all_rejected;
my %unknown;
my %bad;
foreach my $row ( @$rows ) {
    my $id1 = $$row[0];
    my $id2 = $$row[1];
    my $hostname = $$row[2];
    my $path = $$row[3];

    my $name = "$id1.$id2($hostname)";

    unless (defined $path) {
        $bad{$name} = "No path";
        next;
    }

    my $log = "$path.log";

    if ($log =~ /^neb:/) {
        my $new = eval { $neb->find($log); };
        unless ($new) {
            $bad{$name} = $log;
            next;
        }
        $log = $new;
    }

    my $file;                   # Log file
    open $file, $log;
    unless ($file) {
        $bad{$name} = $log;
        next;
    }
    my @log = <$file>;          # Contents of log file
    close $file;
    my $line;                   # Line from file
    my $found;                  # Found something?
    while (($line = pop @log)) {
        if ($line =~ /Assertion .* failed/ or $line =~ /Assertion failed in/ or $line =~ /Aborting in/) {
            $assertion{$name} = $log;
            $found = 1;
            last;
        }
        if ($line =~ /Cannot find file (\S+)/) {
            $cannot_find_file{$name} = $1;
            $found = 1;
            last;
        }
        if ($line =~ /Unable to access file (\S+)/) {
            $unable_access{$name} = $1;
            $found = 1;
            last;
        }
        if ($line =~ /failed to determine real name from template for (\S+)/) {
            $failed_real_name{$name} = $1;
            $found = 1;
            last;
        }
        if ($line =~ /failed to close (\S+)/) {
            $failed_close{$name} = $1;
            $found = 1;
            last;
        }
        if ($line =~ /Unable to open statistics file/) {
            $statistics_file{$name} = $log;
            $found = 1;
            last;
        }
        if ($line =~ /Unable to perform statistics on transparencies./) {
            $transparencies{$name} = $log;
            $found = 1;
            last;
        }
        if ($line =~ /Could not open file,(\S+)/) {
            $couldnt_open_file{$name} = $1;
            $found = 1;
            last;
        }
        if ($line =~ /failed to read (\S+)/) {
            $failed_read{$name} = $1;
            $found = 1;
            last;
        }
        if ($line =~ /no instances available for key: (\S+)/) {
            $no_instances_available{$name} = $1;
            $found = 1;
            last;
        }
        if ($line =~ /neb entry (\S+) not found, not created/) {
            $neb_entry{$name} = $1;
            $found = 1;
            last;
        }
        if ($line =~ /Variance renormalisation is outside valid range: (\S+)/) {
            $variance{$name} = $1;
            $found = 1;
            last;
        }
        if ($line =~ /Error reading .*, (\S+)/) {
            $error_reading{$name} = $1;
            $found = 1;
            last;
        }
        if ($line =~ /All images rejected/) {
            $all_rejected{$name} = $log;
            $found = 1;
            last;
        }
    }
    $unknown{$name} = $log unless $found;
}

print "\n";
print "Total: " . (scalar @$rows) . "\n";
print "\n";

print_failure(\%assertion, "Assertion failures");
print_failure(\%cannot_find_file, "'Cannot find file' errors");
print_failure(\%unable_access, "'Unable to access file' errors");
print_failure(\%failed_real_name, "'Failed to determine real name' errors");
print_failure(\%failed_close, "'Failed to close' errors");
print_failure(\%transparencies, "'Unable to perform statistics on transparencies' errors");
print_failure(\%couldnt_open_file, "'Could not open file' errors");
print_failure(\%failed_read, "'Failed to read' errors");
print_failure(\%no_instances_available, "'No instances available' errors");
print_failure(\%neb_entry, "'Neb entry not found' errors");
print_failure(\%variance, "'Variance renormalisation' errors");
print_failure(\%error_reading, "'Error reading' errors");
print_failure(\%all_rejected, "'All images rejected' errors");
print_failure(\%unknown, "Unknown errors");
print_failure(\%bad, "Bad logs");

### Pau.

sub print_failure
{
    my $data = shift;           # Data
    my $desc = shift;           # Description

    if (scalar keys %$data) {
        print "$desc: " . (scalar keys %$data) . "\n";
        foreach my $name (keys %$data) {
            print "$name: $$data{$name}\n";
        }
        print "\n";
    }
    return 1;
}
