#!/usr/bin/env perl

use warnings;
use strict;
use PS::IPP::Config 1.01 qw( :standard );
use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
use Pod::Usage qw( pod2usage );
use IPC::Cmd 0.36 qw( can_run run );
use DBI;
use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
use File::Temp qw(tempfile);

# globals
my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $label, $singleExpId, $no_publish);

# TODO temporary until we use database to store current jobid
my $jobIdFilePath = "./config/currentJobId.txt";

# default values for certain globals
$verbose = 0;
$save_temps = 0; 
$no_op = 0;
$no_update = 0;
$no_publish = 0;
$camera = 'GPC1';
$output = undef;
$singleExpId = undef;
$datastoreProduct = "PSPS_test";

# get user args
GetOptions(
        'output|o=s' => \$output,
        'batch|b=s' => \$batchType,
        'dvodb|d=s' => \$dvodb,
        'label|l=s' => \$label,
        'expid|e=s' => \$singleExpId,
        'product|p=s' => \$datastoreProduct,
        'no_publish' => \$no_publish,
        'verbose|v' => \$verbose,
        'save_temps' => \$save_temps,
        'no-op' => \$no_op,
        'no-update' => \$no_update,
        ) or pod2usage( 2 );

pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;

# tell off user for not providing proper args
pod2usage(
        -msg => "\n   Required options:\n\n".
        "--batch <init|det|diff|stack>\n".
        "--output <path>\n".
        "--label <label> or --expid <expid>\n" .
        "--dvodb <path>\n".
        "\n   Optional:\n\n".
        "--product <datastoreProduct>\n".
        "--no_publish - won't publish to datastore\n".
        "--save_temps - will save temporary files\n".
        "--no-op - ?\n".
        "--no-update - will not update database\n",
        -exitval => 3
        ) unless 
defined $batchType and
defined $output and
defined $dvodb and
( defined $label or defined $singleExpId );

# check we can run programs and get camera config
my $ippToPsps = can_run('ippToPsps') or (warn "Can't find 'ippToPsps' program" and exit($PS_EXIT_CONFIG_ERROR));
my $dsreg = can_run('dsreg') or (warn "Can't find 'dsreg' program" and exit($PS_EXIT_CONFIG_ERROR));
my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR)); 

# make a temporary file for saving results
my ($resultsFile, $resultsFileName) = tempfile( "/tmp/results.XXXX", UNLINK => !$save_temps);

process();

close $resultsFile;

#######################################################################################
# 
# Generates a PSPS-suitable survey 'type' from the IPP distribution group
#
#######################################################################################
sub getSurveyTypeFromDistGroup {
    my ($distGroup) = @_;

    if ($distGroup eq "MD01") {return $distGroup;}
    if ($distGroup eq "MD02") {return $distGroup;}
    if ($distGroup eq "MD03") {return $distGroup;}
    if ($distGroup eq "MD04") {return $distGroup;}
    if ($distGroup eq "MD05") {return $distGroup;}
    if ($distGroup eq "MD06") {return $distGroup;}
    if ($distGroup eq "MD07") {return $distGroup;}
    if ($distGroup eq "MD08") {return $distGroup;}
    if ($distGroup eq "MD09") {return $distGroup;}
    if ($distGroup eq "MD10") {return $distGroup;}
    if ($distGroup eq "M31") {return $distGroup;}
    if ($distGroup eq "sts") {return "STS";}
    if ($distGroup eq "SweetSpot") {return "SS";}
    if ($distGroup eq "3PI") {return $distGroup;}
    if ($distGroup eq "ThreePi") {return "3PI";}
    if ($distGroup eq "SAS") {return "3PI";}

    print "* Do not understand distribution group: '$distGroup'\n";
    return undef;
}

#######################################################################################
# 
# Finds all exposures for the provided label and generates PSPS FITS data for them
#
#######################################################################################
sub process {

    # Set up the database
    my $dbname = 'gpc1';
    my $dbserver = 'ippdb01';
    my $dbuser = 'ippuser';
    my $dbpass = 'ippuser';
    my $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" .
            "mysql_socket=" . DB_SOCKET(),
            ${dbuser},${dbpass},
            { RaiseError => 1, AutoCommit => 1}
            ) or die "Unable to connect to database $DBI::errstr\n";

    my $query;
    if (!$singleExpId) {

        # query to retrieve all exposures with the provided label
        $query = $db->prepare(<<SQL);

        SELECT DISTINCT rawExp.exp_id, camRun.dist_group
            FROM camRun, chipRun, rawExp
            WHERE camRun.chip_id = chipRun.chip_id
            AND chipRun.exp_id = rawExp.exp_id
            AND camRun.label LIKE '%$label%'
            ORDER BY rawExp.exp_id ASC
SQL
            #AND rawExp.exp_id > 136561
            #AND camRun.dist_group LIKE 'sas'
            #AND rawExp.decl >= '-0.157079633' AND rawExp.decl <= '0.157079633' Jims Dec range
    }
    else {

        $query = $db->prepare(<<SQL);

        SELECT DISTINCT rawExp.exp_id, dist_group FROM rawExp, chipRun WHERE chipRun.exp_id = rawExp.exp_id AND rawExp.exp_id = $singleExpId
SQL
    }

    my $batchId = 0;
    my $batchesPerJob = 1;

    # get a new jobId
    my $jobId = getJobId();
    $jobId++;
    if ($jobId < 0) { print "* Cannot find sensible jobId ($jobId)\n";
        return 0;
    }

    print "*******************************************************************************\n*\n";

    my $anyBatches = 0;


    $query->execute;

    # loop round exposures
    while (my @row = $query->fetchrow_array()) {
        my ($expId, $distGroup) = @row;

        print "* Exposure ID = $expId with dist group = '$distGroup'\n";

        my $surveyType = getSurveyTypeFromDistGroup($distGroup);

        if (!$surveyType) {next;}
        print "* Using survey type of $surveyType\n";

        # generate batch and job paths from job and batch IDs
        my $job = sprintf("J%06d", $jobId);
        my $jobOutputPath = sprintf("$output/$job");
        my $batchDir = sprintf("B%03d", $batchId);
        my $batchOutputPath = sprintf("$jobOutputPath/%s", $batchDir);

        print "* Preparing exposure $expId as job '$job'...\n";

        # make directories
        mkdir($jobOutputPath, 0777);
        mkdir($batchOutputPath, 0777);
        setJobId($jobId);

        # run IppToPsps program
        if (runIppToPsps($db, $expId, $batchOutputPath, $batchType)) {

            # write batch manifest and tar and zip up
            if (writeBatchManifest($batchOutputPath, $batchDir, $batchType)) {

                if (tarAndZipDirectory($jobOutputPath, $batchDir)) {

                    $anyBatches = 1
                }
            }

            if (($batchId+1) < $batchesPerJob) {
                $batchId++;
            }
            # job is complete, now publish
            elsif ($anyBatches) {

                if (writeJobManifest($output, $job, $surveyType)) {

                    if (!$no_publish && !publishToDatastore($output, $job)) { 

                    }
                }

                $jobId++;
                $batchId = 0;
                $anyBatches = 0;
            }
            else {

                print "* No batches to publish\n"
            }
        }

        if ($batchType eq "init" ) {print "* Wrote initialisation batch\n"; last;}


        print "*\n*******************************************************************************\n\n";
    }

    $db->disconnect();

    return 1;
}

#######################################################################################
#
# Gets the current jobId (ini file for now - will be Db eventually TODO) 
#
########################################################################################
sub getJobId {

    my $jobId = -1;

    open (MYFILE, $jobIdFilePath);
    while (<MYFILE>) {
        chomp;
        $jobId = $_;
    }
    close (MYFILE);

    if ($verbose) { print "* Current jobId = $jobId\n"; }

    return $jobId;
}

#######################################################################################
#
# Sets the current jobId (in file for now - will be Db eventually TODO) 
#
########################################################################################
sub setJobId {
    my ($jobId) = @_;

    open (MYFILE, "> $jobIdFilePath");
    print MYFILE "$jobId";
    close (MYFILE);

    if ($verbose) { print "* New jobId = $jobId\n"; }
}

#######################################################################################
#
#run IppToPsps 
#
#######################################################################################
sub runIppToPsps {
    my ($db, $expId, $batchOutputPath, $batchType) = @_;

    my $success = 0;

    # before anything else, we can publish the init batch
    if ($batchType =~ /init/) {
        $success = produceInit($batchOutputPath);
    }
    elsif ($batchType =~ /det/) {
        $success = produceDetections($db, $expId, $batchOutputPath);
    }
    elsif ($batchType =~ /diff/) {
        $success = produceDiffs($db, $expId, $batchOutputPath);
    }

    return $success;
}

#######################################################################################
#
# tar 'n zip up a directory TODO handle errors
#
#######################################################################################
sub tarAndZipDirectory {
    my ($jobOutputPath,$batch) = @_;

    my $batchDir = "$jobOutputPath/$batch";
    my $batchTar = "$batchDir.tar";
    my $batchZip = "$batchTar.gz";

    # tar
    my $command = "tar -cf $batchTar -C $jobOutputPath $batch";
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run(command => $command, verbose => $verbose);

    if (!$success) { print "* Unable to tar up dir: $batch\n" and return 0 };

    # zip
    $command = "gzip -c $batchTar  > $batchZip";
    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run(command => $command, verbose => $verbose);

    if (!$success) { print "* Unable to gzip: $batchTar\n" and return 0 };

    $command = "rm $batchTar";
    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run(command => $command, verbose => $verbose);

    if (!$success) { print "* Unable to remove: $batchTar\n" };

    $command = "rm -r $batchDir";
    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run(command => $command, verbose => $verbose);

    if (!$success) { print "* Unable to remove: $batch\n"};

    return 1;
}

#######################################################################################
#
# writes a batch manifest file for the one FITS file that should be in the path passed in
#
#######################################################################################
sub writeBatchManifest {

    my ($path,$batch,$batchType) = @_;

    use XML::Writer;
    use Digest::MD5::File qw( file_md5_hex );

    # sort out paths
    my $outputPath = "$path/BatchManifest.xml";
    my $output = new IO::File(">$outputPath");

    # create a timestamp
    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
    my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec;

    # determine batch 'type'
    my $type;
    if($batchType eq 'init') {$type = "IN";}
    elsif($batchType eq 'det') {$type = "P2";}
    elsif ($batchType eq 'stack') {$type = "ST";}
    elsif ($batchType eq 'diff') {$type = "OB";}
    else {$type = "UNKNOWN";}

    # read results
    open (MYFILE, $resultsFileName);
    my $i = 0;

    # detection results
    my $minObjId;
    my $maxObjId;
    my $filename;
    if($batchType eq 'det') {

        while (<MYFILE>) {
            chomp;
            if ($i == 0) { $filename = $_; }
            if ($i == 1) { $minObjId = $_; }
            if ($i == 2) { $maxObjId = $_; }
            $i++;
        }
    }
    close (MYFILE);

    # create XML file
    my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2);
    $writer->xmlDecl('UTF-8');
    $writer->doctype('manifest', "", "psps-manifest.dtd");

    # open directory to hunt for FITS files
    opendir(DIR, $path) or print "* Cannot open '$path' to write batch manifest\n" and return 0;
    my @thefiles= readdir(DIR);
    closedir(DIR);

    my $foundFits = 0;

    foreach my $f (@thefiles) {

        if ($f =~ /\.FITS/) {

            if ($foundFits) { print "* More than one FITS file found for this batch\n"; return 0;}

            # get md5sum of file
            my $md5sum  = file_md5_hex("$path/$f");

            # get size of file in bytes
            my $filesize = -s "$path/$f";

            # write manifest element TODO untidy
            if($batchType eq 'det' && $filename eq $f) {

                $writer->startTag('manifest',
                        "name" => "$batch",
                        "type" => $type,
                        "timestamp" => "$timeStamp",
                        "minObjId" => $minObjId,
                        "maxObjId" => $maxObjId);
            }
            else {

                $writer->startTag('manifest',
                        "name" => "$batch",
                        "type" => $type,
                        "timestamp" => "$timeStamp");
            }
            # write the tag for this batch
            $writer->startTag("file", 
                    "name" => $f,
                    "bytes" => $filesize,
                    "md5" => $md5sum);

            #$writer->startTag('file' );
            #$writer->startTag('name'); $writer->characters("$f"); $writer->endTag();
            #$writer->startTag('bytes'); $writer->characters("$filesize"); $writer->endTag();
            #$writer->startTag('md5'); $writer->characters("$md5sum"); $writer->endTag();

            #if($batchType eq 'det' && $filename eq $f) {

                #    $writer->startTag('minObjId'); $writer->characters("$minObjId"); $writer->endTag();
                #    $writer->startTag('maxObjId'); $writer->characters("$maxObjId"); $writer->endTag();
                #}
            $writer->endTag();
            # finish up
            $writer->endTag();
            $writer->end();
            $foundFits = 1;
        }

    }


    if (!$foundFits) { print "* Could not find any FITS files for this batch\n"; return 0;}
    return 1;
}

#######################################################################################
#
# write job manifest
#
#######################################################################################
sub writeJobManifest {
    my ($path, $job, $surveyType) = @_;

    use XML::Writer;
    use Digest::MD5::File qw( file_md5_hex );

    # sort out paths
    my $outputPath = "$path/$job/JobManifest.xml";
    my $output = new IO::File(">$outputPath");

    # create a timestamp
    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
    my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec;

    # create XML file
    my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2);
    $writer->xmlDecl('UTF-8');
    $writer->doctype('manifest', "", "psps-manifest.dtd");
    $writer->startTag('manifest',
            "name" => "$job",
            "type" => $surveyType,
            "timestamp" => "$timeStamp");

    my $jobPath = "$path/$job";

    # open directory to hunt for FITS files
    opendir(DIR, $jobPath) or print "* Cannot open '$path' to write job manifest\n" and return 0;
    my @thefiles= readdir(DIR);
    closedir(DIR);

    my $foundBatch = 0;

    foreach my $f (@thefiles) {

        if (($f =~ /\.tar\.gz/)) {

            my $filePath = "$jobPath/$f";

            # get md5sum of file
            my $md5sum  = file_md5_hex($filePath);

            # get size of file in bytes
            my $filesize = -s $filePath;

            # write the tag for this batch
            $writer->startTag('file',
                    "name" => $f,
                    "bytes" => "$filesize",
                    "md5" => "$md5sum",
                    );
            $writer->endTag();
            $foundBatch = 1;
        }
    }

    # finish up
    $writer->endTag();
    $writer->end();

    if (!$foundBatch) { print "* Could not find any batches for this job\n"; return 0;}
    return 1;
}

#######################################################################################
#
# register new job with the datastore
#
########################################################################################
sub publishToDatastore {
    my ($path, $job) = @_;

    my $fullJobPath = "$path/$job";

    my ($tempFile, $tempName) = tempfile( "/tmp/dsregList.XXXX", UNLINK => !$save_temps);

    # loop through all batch files in this job directory
    opendir(DIR, $fullJobPath) or print "* Cannot open '$fullJobPath' in order to publish to datastore\n" and return 0;
    my @thefiles= readdir(DIR);
    closedir(DIR);

    my $foundBatch = 0;

    foreach my $f (@thefiles) {

        # get all batch tgz files
        if ($f =~ /\.tar.gz/) {print $tempFile $f . '|||tgz' . "\n"; $foundBatch = 1;}

        # get JobManifest XML file
        elsif ($f =~ /\.xml/) {print $tempFile $f . '|||xml' . "\n";}
    }

    if (!$foundBatch) { print "Could not find any batches to publish\n"; return 0;}

    # build dsreg command command
    my $command  = "$dsreg";
    $command .= " --add $job";
    $command .= " --copy";
    $command .= " --datapath $fullJobPath";
    $command .= " --type IPP_PSPS";
    $command .= " --product $datastoreProduct";
    $command .= " --list $tempName";

    # run command
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
        run(command => $command, verbose => $verbose);

    if (!$success) { print "* Unable to publish $job to datastore\n" and return 0 };

    print "* Successfully published $job to datastore\n";

    close($tempFile);

    return 1;
}

#######################################################################################
# 
# runs ippToPsps to produce init batch 
#
#######################################################################################
sub produceInit {
    my ($outputPath) = @_;

    my $success = runProgram( "NULL", $outputPath, 0, $batchType);
    return $success;
}
#######################################################################################
# 
# runs ippToPsps for detections for this exposure 
#
#######################################################################################
sub produceDetections {
    my ($db,$expId,$outputPath) = @_;

    # query to retrieve nebulous key of camera smf file for this exposure
    my $query = 
        "SELECT camProcessedExp.path_base ".
        "FROM warpRun ".
        "JOIN fakeRun USING(fake_id) ".
        "JOIN camRun USING(cam_id) ".
        "JOIN camProcessedExp USING(cam_id) ".
        "JOIN chipRun USING(chip_id) ".
        "JOIN rawExp USING (exp_id) ".
        "WHERE rawExp.exp_id = $expId AND camRun.magicked";

    if ($verbose) { print"* $query\n";}


    # execute query and check results
    my $results = $db->selectall_arrayref( $query );
    my $size = @$results;
    if ($size < 1) {print "* No smf files found for this exposure\n"; return 0;}

    # loop round db results TODO should only be one result
    my $success = 0;
    for my $row (@$results) {

        # get real filename from neb 'key'
        my ($nebPath) = @$row;
        my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",$nebPath) or return 0;
        my $realFile = $ipprc->file_resolve($fpaObjects) or return 0;

        # now write the path to this file out to temp file
        my ($tempFile, $tempName) = tempfile( "/tmp/inputFileList.XXXX", UNLINK => !$save_temps);
        print $tempFile $realFile . "\n";
        close $tempFile;

        $success = runProgram($tempName, $outputPath, $expId, $batchType);
        return $success;
    }
}

#######################################################################################
# 
# runs ippToPsps for diffs for this exposure 
#
#######################################################################################
sub produceDiffs {
    my ($db,$expId,$outputPath) = @_;

    my $query = 
        "SELECT DISTINCT diffSkyfile.path_base ".
        "FROM warpRun ".
        "JOIN fakeRun USING(fake_id) ".
        "JOIN camRun USING(cam_id) ".
        "JOIN chipRun USING(chip_id) ".
        "JOIN rawExp USING (exp_id) ".
        "JOIN diffInputSkyfile ON (warpRun.warp_id = diffInputSkyfile.warp1 OR warpRun.warp_id = diffInputSkyfile.warp2) ".
        "JOIN diffSkyfile USING (diff_id) ".
        "WHERE rawExp.exp_id = $expId AND camRun.magicked";

    if ($verbose) { print"* $query\n";}

    # execute query and check results
    my $results = $db->selectall_arrayref( $query );
    my $size = @$results;
    if ($size < 1) {print "* No cmf files found for this exposure\n"; return 0;}

    my ($tempFile, $tempName) = tempfile( "inputFileList.XXXX", UNLINK => !$save_temps);

    # loop round db results should be one file per skycell
    my $success = 0;
    for my $row (@$results) {

        my ($nebPath) = @$row;
        #print "NEB PATH $nebPath\n";

        # get real filename from neb 'key'
        my $fpaObjects = $ipprc->filename("PPSUB.OUTPUT.SOURCES",$nebPath) or return 0;
        my $realFile = $ipprc->file_resolve($fpaObjects) or return 0;
        #print "REALFILE $realFile\n";

        # now write the path to this file out to temp file
        print $tempFile $realFile . "\n";
    }

    my $ret = runProgram($tempName, $outputPath, $expId, $batchType);

    close $tempFile;

    return $ret;
}

#######################################################################################
#
# runs the ippToPsps program
#
#######################################################################################
sub runProgram {
    my ($input, $output, $expid, $batchType) = @_;

    # build command
    my $command = "$ippToPsps"; 
    $command .= " -input $input";
    $command .= " -output $output";
    $command .= " -D CATDIR $dvodb";
    $command .= " -config config"; # TODO
    $command .= " -expid $expid";
    $command .= " -batch $batchType";
    $command .= " -results $resultsFileName";

    # run command
    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf )
        = run(command => $command, verbose => $verbose);

    return $success;
}



