#!/usr/bin/perl -w

package czartool::Plotter;

use warnings;
use strict;

use File::Temp;

use czartool::Config;

my @allStages = (
        "chip",
        "cam",
        "fake",
        "warp",
        "stack",
        "staticsky",
        "diff",
        "dist",
        "pub",
        "chipbackground");

my @allIppToPspsStages = (
        "processed",
        "loaded_to_datastore",
        "loaded_to_ODM",
        "merge_worthy",
        "deleted_datastore",
        "deleted_dxlayer",
        "merged",
        "deleted_local");

my $GNUPLOT="/home/panstarrs/ipp/local/bin/gnuplot";

###########################################################################
#
# Main constructor with all arguments
#
###########################################################################
sub new {
    my $class = shift;
    my $self = {
        _config => shift,
        _dateFormat => shift,
        _outputFormat => shift,
        _outputPath => shift,
        _save_temps => shift,
    };

    bless $self, $class;
    $self->init();
    return $self;
}

###########################################################################
#
# Constructor sending all plots to file
#
###########################################################################
sub new_file {
    my $class = shift;
    my $self = {
        _config => shift,
        _outputPath => shift,
        _save_temps => shift,
                                                    };
    $self->{_dateFormat} = "%Y%m%d-%H%M%S";
    $self->{_outputFormat} = "png font \"".$self->{_config}->getGnuplotFont()."\" ".$self->{_config}->getGnuplotFontSize();

    bless $self, $class;
    $self->init();
    return $self;
}

###########################################################################
#
# Constructor sending all plots to an X window
#
###########################################################################
sub new_display {
    my $class = shift;
    my $self = {
        _config => shift,
        _save_temps => shift,
    };

    $self->{_dateFormat} = "%Y%m%d-%H%M%S";
    $self->{_outputFormat} = "X11";
    $self->{_outputPath} = undef;

    bless $self, $class;
    $self->init();
    return $self;
}

###########################################################################
#
# Sets plotter to use standard IPP mode, i.e. chip, cam etc
#
###########################################################################
sub setStandardMode {
    my ($self) = @_;

    $self->{allStages} = \@allStages;
}

###########################################################################
#
# Sets plotter to use ippToPsps mode, i.e. processed, loaded_to_datastore etc
#
###########################################################################
sub setIppToPspsMode {
    my ($self) = @_;

    $self->{allStages} = \@allIppToPspsStages;
}

###########################################################################
#
# Some initialisation code common to all constructors
#
###########################################################################
sub init {
    my ($self) = @_;

    $self->{_gpc1Db} = $self->{_config}->getGpc1Instance();
    $self->{_czarDb} = $self->{_config}->getCzarDbInstance();
    $self->setStandardMode();
}

###########################################################################
#
# Generates a suitable filename for this plot
#
###########################################################################
sub createImageFileName {
    my ($self, $label, $stage, $suffix, $isLog, $isRate) = @_;

    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : ".";
    my $stagePart = $stage ? $stage : "all_stages";
    my $type = $isLog ? "log" : "linear";
    return $prefix . "/czarplot_" . $type . "_" . $label . "_" . $stagePart . "_$suffix.png";
}

###########################################################################
#
# Figures out a suitable interval for rate plots
# Could use a function here, but instead we want to choose meaningful intervals
#
###########################################################################
sub getSuitableInterval {
    my ($self, $begin, $end, $interval) = @_;

    my $timeDiff = $self->{_czarDb}->diffTimes($end, $begin);

    if ($self->{_czarDb}->isBefore($timeDiff, "00:00:01")) {return 0;}
    elsif ($self->{_czarDb}->isBefore($timeDiff, "03:00:00")) {${$interval} = "15 MINUTE";}
    elsif ($self->{_czarDb}->isBefore($timeDiff, "10:00:00")) {${$interval} = "30 MINUTE";}
    elsif ($self->{_czarDb}->isBefore($timeDiff, "26:00:00")) {${$interval} = "1 HOUR";} # under 1 day
    elsif ($self->{_czarDb}->isBefore($timeDiff, "36:00:00")) {${$interval} = "2 HOUR";} # under 1.5 days
    elsif ($self->{_czarDb}->isBefore($timeDiff, "48:00:00")) {${$interval} = "3 HOUR";} # under 2 days
    elsif ($self->{_czarDb}->isBefore($timeDiff, "72:00:00")) {${$interval} = "4 HOUR";} # under 3 days
    elsif ($self->{_czarDb}->isBefore($timeDiff, "96:00:00")) {${$interval} = "5 HOUR";} # under 4 days
    elsif ($self->{_czarDb}->isBefore($timeDiff, "144:00:00")) {${$interval} = "6 HOUR";} # under 6 days
    elsif ($self->{_czarDb}->isBefore($timeDiff, "168:00:00")) {${$interval} = "12 HOUR";} # under week
    elsif ($self->{_czarDb}->isBefore($timeDiff, "672:00:00")) {${$interval} = "1 DAY";} # under 2 week
    elsif ($self->{_czarDb}->isBefore($timeDiff, "2016:00:00")) {${$interval} = "1 WEEK";} # under 3 month
    else {${$interval} = "1 MONTH";}

# ${$interval} = "2 HOUR";

    return 1;
}

###########################################################################
#
# Plots a time series of processing rate for all stages for this label TODO duplication below. combine
#
###########################################################################
sub createRateHistogram {
    my ($self, $label, $selectedStage, $beginTime, $endTime, $interval) = @_;

    # stages
    my $stages = undef;
    if (!$selectedStage) {
        $stages = \@{$self->{allStages}};
        $selectedStage = 'all_stages';
    }
    else {$stages = ["$selectedStage"]};

    # interval
    if (!$interval && !$self->getSuitableInterval($beginTime, $endTime, \$interval)) {return 0;}

    # set up files
    my %gnuplotFiles;
    my $stage = undef;
    my $gnuplotFile = undef;
    my $minX = 999999999;
    my $maxX = -9999999999;
    my $timeDiff = 0;

    foreach $stage (@{$stages}) {

        if ($self->{_czarDb}->createProcessingRateData(
                    $stage,
                    $label,
                    $beginTime,
                    $endTime,
                    $interval,
                    \$gnuplotFile,
                    \$minX, \$maxX, \$timeDiff)) {

            $gnuplotFiles{$stage} = $gnuplotFile;
            #print "$minX, $maxX, $timeDiff\n";
        }
    }

    my $outputFile = createImageFileName($self, $label, $selectedStage, "rh", 0);
    $self->plotRateStackedHistogram(\%gnuplotFiles, $outputFile, $label, $selectedStage, $beginTime, $endTime, $interval);

    # now delete temp dat files
    foreach my $stage (keys %gnuplotFiles) {unlink($gnuplotFiles{$stage});}

    return 1;
}

###########################################################################
#
# Plots a stacked histogram of rate data for one or more stages
#
###########################################################################
sub plotRateStackedHistogram {
    my ($self, $gnuplotFiles, $outputFile, $label, $selectedStage, $beginTime, $endTime, $interval) = @_;

    open (GP, "|$GNUPLOT -persist") or die "no gnuplot";
    use FileHandle;
    GP->autoflush(1);

    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}

    print GP
        "set term $self->{_outputFormat};" .
        "set title \"'$label' for '$selectedStage'\\nFrom '$beginTime' to '$endTime' HST\";" .
        "set boxwidth;" .
        "set xtic rotate by -90 scale 0;" .
        "set style data histogram;" .
        "set style histogram rowstacked;" .
        "set style fill solid border -1;" .
        "set ylabel \"Items processed per $interval\";" .
        "set boxwidth 0.75;" .
        "plot ";

    my $first = 1;
    foreach my $stage (@{$self->{allStages}}) {


        if(!$gnuplotFiles->{$stage}) {next;}

        if (!$first) { print GP ","; }
        print GP "'" . $gnuplotFiles->{$stage}. "' using 4:xtic(1) title \"$stage\" ";
        $first = 0;
    }

    print GP ";\n";
    close GP;

    return 1;
}

###########################################################################
#
# Plots a time series for all or one stage for this label
#
###########################################################################
sub createTimeSeries {
    my ($self, $label, $selectedStage, $beginTime, $endTime, $linear, $log, $rate) = @_;

    my $minX = 999999999;
    my $maxX = -9999999999;
    my $timeDiff = 0;

    my $stages = undef;

    if (!$selectedStage) {$stages = \@{$self->{allStages}};}
    else {$stages = ["$selectedStage"]};

    my %linDataFiles;
    my %logDataFiles;
    my %ratDataFiles;
    my $stage = undef;
    my $linDataFile = undef;
    my $logDataFile = undef;
    my $ratDataFile = undef;

    foreach $stage (@{$stages}) {

        if($self->{_czarDb}->createTimeSeriesData(
                    $label,
                    $stage,
                    $beginTime,
                    $endTime,
                    \$minX,
                    \$maxX,
                    \$timeDiff,
                    \$linDataFile,
                    \$logDataFile,
                    \$ratDataFile)) {

            $linDataFiles{$stage} = $linDataFile;
            $logDataFiles{$stage} = $logDataFile;
            $ratDataFiles{$stage} = $ratDataFile;
        }
    }

    my $numOfPlots =  keys(%linDataFiles);

    if ($numOfPlots == 0 ) {

#        print "Warning: No plots could be generated for stage '$selectedStage' and label '$label' during time period '$beginTime', '$endTime'\n";
        return;
    }

    if ($timeDiff == 0) {

        print "WARNING: Zero time difference for '$label' during time period '$beginTime', '$endTime'\n";
        return;
    }

    if ($linear) {

        my $outputFile = createImageFileName($self, $label, $selectedStage, "t", 0);

        $self->plotTimeSeries(
                \%linDataFiles,
                $outputFile,
                $label,
                $beginTime,
                $endTime,
                $maxX,
                $minX,
                $timeDiff,
                "",
                "Items");
    }

    if ($log) {

        my $outputFile = createImageFileName($self, $label, $selectedStage, "t", 1);

        $self->plotTimeSeries(
                \%logDataFiles,
                $outputFile,
                $label,
                $beginTime,
                $endTime,
                $maxX,
                $minX,
                $timeDiff,
                "",
                "Log(Items)");
    }

    if ($rate) {

        my $outputFile = createImageFileName($self, $label, $selectedStage, "rt", 0);

        $self->plotTimeSeries(
                \%ratDataFiles,
                $outputFile,
                $label,
                $beginTime,
                $endTime,
                $maxX,
                $minX,
                $timeDiff,
                "",
                "Items per hour");
}
}

###########################################################################
#
# Plots a histogram of stuff processed in provided interval and label for all stages
#
###########################################################################
sub createHistogram {
    my ($self, $label, $beginTime, $endTime) = @_;

    my $outputFile = createImageFileName($self, $label, undef, "h", 0);

    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_".$label."_h.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
    open (GNUDAT, ">".$tmpFile->filename);

    my ($processed, $pending, $faults);
    my $stage = undef;
    my $pendingMinusFaults = undef;
    foreach $stage (@{$self->{allStages}}) {

        if (!$self->{_czarDb}->countProcessedPendingAndFaults(
                $label,
                $stage,
                $beginTime,
                $endTime,
                \$processed,
                \$pending,
                \$faults)) {next;}

        $pendingMinusFaults = $pending - $faults;
        print GNUDAT "$stage $faults $processed $pendingMinusFaults\n";
    }

    close(GNUDAT);

    open (GP, "|$GNUPLOT -persist") or die "no gnuplot";
    use FileHandle;
    GP->autoflush(1);

    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
    print GP
        "set term $self->{_outputFormat};" .
        "set title \"'$label'\\nFrom '$beginTime' to '$endTime' HST\";" .
        "set grid;" .
        "set boxwidth;" .
        "set style data histogram;" .
        "set style histogram rowstacked;" .
        "set style fill solid border -1;" .
        "set ylabel \"Items\";" .
        "set boxwidth 0.75;" .
        "plot '".$tmpFile->filename."' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" .
        "\n";

    close GP;
}

###########################################################################
#
# Sets the output path
#
###########################################################################
sub setOutputFormat {
    my ($self, $outputFormat) = @_;
    $self->{_outputFormat} = $outputFormat if defined($outputFormat);
}

###########################################################################
#
# Sets the output type
#
###########################################################################
sub setOutputPath {
    my ($self, $outputPath) = @_;
    $self->{_outputPath} = $outputPath if defined($outputPath);
}

###########################################################################
#
# Sets the date format to be used
#
###########################################################################
sub setDateFormat {
    my ($self, $dateFormat) = @_;
    $self->{_dateFormat} = $dateFormat if defined($dateFormat);
    return $self->{_dateFormat};
}

###########################################################################
#
# Figures out suitable spacing for time tics on time-series plots
#
###########################################################################
sub getTimeSpacing {
    my ($self, $timeDiff, $divX, $timeFormat) = @_;

    # if less than a few mins of data, show 1 minute tics
    if ($timeDiff < 300) {
        ${$timeFormat} = "%H:%M";
        ${$divX} = 60;
    }
    # if less than a few mins of data, show 5 minute tics
    elsif ($timeDiff < 1800) {
        ${$timeFormat} = "%H:%M";
        ${$divX} = 300;
    }
    # if less than an hour of data, show 15 minute tics
    elsif ($timeDiff < 3600) {
        ${$timeFormat} = "%H:%M";
        ${$divX} = 900;
    }
    # if less than a couple of hour's data plotted, show 30 mins tics
    elsif ($timeDiff < 7200) {
        ${$timeFormat} = "%H:%M";
        ${$divX} = 1800;
    }
    # if less than half a day's data plotted, show hourly tics
    elsif ($timeDiff < 43200) {
        ${$timeFormat} = "%H:%M";
        ${$divX} = 3600;
    }
    # if less than one day's data plotted, show 2 hourly tics
    elsif ($timeDiff < 86400) {
        ${$timeFormat} = "%H:%M";
        ${$divX} = 7200;
    }
    # if less than 1 week's data is data plotted, show daily tics
    elsif ($timeDiff < 604800) {

        ${$timeFormat} = "%m/%d";
        ${$divX} = 86400;
    }
    # if less than 1 months's data is data plotted, show weekly ticks
    elsif ($timeDiff < 2419200) {

        ${$timeFormat} = "%m/%d";
        ${$divX} = 604800;
    }
    # otherwise, show monthly ticks
    else {

        ${$timeFormat} = "%b";
        ${$divX} = 2419200;
    }
}

###########################################################################
#
# Plots a time-series of pending/faults
#
###########################################################################
sub plotTimeSeries {
    my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $timeDiff, $title, $yTitle) = @_;

    my $timeFormat = undef;
    my $divX = undef;

    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);

    my $numOfPlots = keys %$gnuplotFiles;

    # sort out plot title
    if ($numOfPlots == 1) {foreach my $stage (keys %$gnuplotFiles) {$title .= "'".$stage."'";}}
    else {$title .= "'all stages'"}

    $title .= " for '$label'\\nFrom '$fromTime' to '$toTime' HST";

    open (GP, "|$GNUPLOT -persist") or die "no gnuplot";
    use FileHandle;
    GP->autoflush(1);

    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
    print GP
        "set term $self->{_outputFormat};" .
        "set title \"$title\";" .
        "set key left top;" .
        "set xdata time;" .
        "set timefmt \"$self->{_dateFormat}\";" .
        "set format x \"$timeFormat\";" .
        "set xtics \"$minX\", $divX, \"$maxX\";" .
        "set grid xtics;" .
        "set grid ytics;" .
        "set xlabel \"Time\";" .
        "set ylabel \"$yTitle\";" .
        "plot ";

    my $firstIn = 1;
    # loop through stages array so that they are ordered properly (not maintained in hash)
    #foreach my $stage (@allStages) {
    foreach my $stage (@{$self->{allStages}}) {

        if(!$gnuplotFiles->{$stage}) {next;}
        if (!$firstIn) {print GP ",";}

        # for single stage plots, show pending/processed/faults
        if ($numOfPlots == 1) {

            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"Pending\" with lines lt 4 lw 2,";
            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"Processed\" with lines lt 2 lw 2,";
            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"Faults\" with lines lt 1 lw 2";
        }
        # when plotting multiple stages, show only processed
        else {

            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"$stage\" with lines lw 2";
        }
        $firstIn = 0;
    }

    print GP "\n";
    close GP;

    # now delete temp dat files
    foreach my $stage (keys %$gnuplotFiles) {unlink($gnuplotFiles->{$stage});}
}

###########################################################################
#
# Plots a time-series of cluster storage
#
###########################################################################
sub plotStorageTimeSeries {
    my ($self, $fromTime, $toTime) = @_;

    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : ".";
    my $outputFile = "$prefix/czarplot_cluster.png";
    my ($minX, $maxX, $minY, $maxY, $timeDiff);

    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_gnuplot_storage_timeseries.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
    if (!$self->{_czarDb}->createStorageTimeSeriesData($tmpFile, $fromTime, $toTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff)) {return 0;}

    my $timeFormat = undef;
    my $divX = undef;

    $self->getTimeSpacing($timeDiff, \$divX, \$timeFormat);

    open (GP, "|$GNUPLOT -persist") or die "no gnuplot";
    use FileHandle;
    GP->autoflush(1);

    my $datFile = $tmpFile->filename;
    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
    print GP <<PLOT;
        set term $self->{_outputFormat}
        set title "Total available cluster space over time\\nFrom $fromTime to $toTime HST"
        set key left top
        set xdata time
        set timefmt "$self->{_dateFormat}"
        set format x "$timeFormat"
        set xtics "$minX", $divX, "$maxX"
        set grid xtics
        set xlabel "Time"
        set ylabel "Available (TB)"
        plot "$datFile" using 1:2 title "Available" with lines lt 2 lw 2, "$datFile" using 1:3 title "Theoretical" with lines lt 3 lw 2
PLOT

    print GP "\n";
    close GP;

    return 1;
}

###########################################################################
#
# Plots disk usage across cluster as a stacked histogram
#
###########################################################################
sub plotDiskUsageHistogram {
    my ($self) = @_;

    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
        my $outputFile = "$prefix/czarplot_hosts_space.png";
    my $limit = 97.0;
    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_hosts_space.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
    $self->{_czarDb}->createHostsData($limit, $tmpFile);

    my $totalUsed = $self->{_czarDb}->getTotalClusterStorageAsPercentage();

    my $totalPercent = sprintf("%.1f%%", $totalUsed);

    open (GP, "|$GNUPLOT -persist") or die "no gnuplot";
    use FileHandle;
    GP->autoflush(1);

    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
    print GP
        "set term $self->{_outputFormat};" .
        "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" .
        "set style fill solid 1.00 border -1;" .
        "set key center top;" .
        "set style histogram rowstacked;" .
        "set style data histograms;" .
        "set ylabel \"Space (TB)\";" .
        "set xtic rotate by -90 scale 0;" .
	"set yrange [:100];" .
        "plot '".$tmpFile->filename."' " .
        "using 2:xtic(1) t \"Used\" lt 7," .
        "'' using 3 t \"Over $limit% used\" lt 1," .
        "'' using 4 t \"Unavailable\" fs solid 0.50 lt -1 ," .
        "'' using 5 t \"Free\" lt 2," .
        "'' using 6 notitle fs solid 0.50 lt -1" .
        ";" .

        "\n";
    close GP;
}

###########################################################################
#
# Plots disk usage across cluster as a stacked histogram
#
###########################################################################
sub plotDiskUsageHistogramLarge {
    my ($self) = @_;

    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
        my $outputFile = "$prefix/czarplot_hosts_space_large.png";
    my $limit = 97.0;
    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_hosts_space.XXXXX", DIR => '/tmp', SUFFIX => 'dat');
    $self->{_czarDb}->createHostsData($limit, $tmpFile);

    my $totalUsed = $self->{_czarDb}->getTotalClusterStorageAsPercentage();

    my $totalPercent = sprintf("%.1f%%", $totalUsed);

    open (GP, "|$GNUPLOT -persist") or die "no gnuplot";
    use FileHandle;
    GP->autoflush(1);

    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
    print GP
        "set term $self->{_outputFormat} size 2560,1024;" .
        "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" .
        "set style fill solid 1.00 border -1;" .
        "set bmarg 5; " .
        "set key center top;" .
        "set style histogram rowstacked;" .
        "set style data histograms;" .
        "set ylabel \"Space (TB)\";" .
	"set yrange [:100];" .
        "set xtic rotate by -90 scale 0;" .
        "plot '".$tmpFile->filename."' " .
        "using 2:xtic(1) t \"Used\" lt 7," .
        "'' using 3 t \"Over $limit% used\" lt 1," .
        "'' using 4 t \"Unavailable\" fs solid 0.50 lt -1 ," .
        "'' using 5 t \"Free\" lt 2," .
        "'' using 6 notitle fs solid 0.50 lt -1" .
        ";" .

        "\n";
    close GP;
}

###########################################################################
#
# Generates 2D plot of magic mask fraction for provided exposure ID
#
###########################################################################
sub plotMagicMaskFractionForThisExposure {
    my ($self, $exp_id) = @_;

    # set up file stuff
    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
        my $outputFile = "$prefix/czarplot_single_exp_magic_mask_fraction.png";
    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_single_exp_magic_mask_fraction.XXXXX", DIR => '/tmp', SUFFIX => '.dat');

    my $fracs;
    if (!$self->{_gpc1Db}->getMagicMaskFraction($exp_id, \$fracs)) {return 0;}

    open (GNUDAT, ">".$tmpFile->filename);

    # missing corner chip
    print GNUDAT "0 0 0.0\n";
    foreach my $row ( @{$fracs} ) {

        if (@{$row}[0] =~ m/XY([0-9])([0-9])/) {
            print GNUDAT "$1 $2 @{$row}[1]\n";
            # missing corner chips
            if($1 == 0 && $2 == 6) {print GNUDAT "0 7 0.0\n";}
            if($1 == 6 && $2 == 7) {print GNUDAT "7 0 0.0\n";}
        }
    }
    # missing corner chip
    print GNUDAT "7 7 0.0\n";


    close(GNUDAT);


    open (GP, "|$GNUPLOT -persist") or die "no gnuplot";
    use FileHandle;
    GP->autoflush(1);

    my $datFile = $tmpFile->filename;

    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$outputFile\";";}
    print GP <<PLOT;

    set term $self->{_outputFormat}
    set title "Magic mask fraction for exposure $exp_id"
        set grid
        set size square
        set xrange [-0.5:7.5] reverse
        set yrange [-0.5:7.5]
        unset key
        set palette rgb 22,13,10
        set ylabel "Y"
        set xlabel "X"
        set cbrange [0:1]
        set datafile missing "NaN"
        plot "$datFile" using 1:2:3  with image
PLOT

    close GP;

    return 1;
}


###########################################################################
#
# Plots mask fraction histogram for exposures taken between these dates
#
###########################################################################
sub plotMagicMaskFraction {
    my ($self, $begin, $end) = @_;

    # get exposures between these dates
    my $exp_ids = undef;
    if (!$self->{_gpc1Db}->getProcessedExposures($begin, $end, \$exp_ids)) {return 0;}
    my $totalExp = @{$exp_ids};
    if ($totalExp < 1) {return 0;}

    # set up file stuff
    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : "."; # TODO should be function for this
        my $histoOutputFile = "$prefix/czarplot_magic_mask_fraction_h.png";
    my $distOutputFile = "$prefix/czarplot_magic_mask_fraction_d.png";
    my $tmpFile = File::Temp->new( TEMPLATE => "czarplot_magic_mask_fraction.XXXXX", DIR => '/tmp', SUFFIX => 'dat');

    # set up bins
    my %histogram;
    my @bins;
    my $interval = 0.05;
    my $min = 1.0;
    my $max = 1.0;
    for (my $n=0.0; $n<$max; $n += $interval){

        push(@bins, $n);
        $histogram{$n} = 0;
    }

    my $meanMask = undef;
    my $sumMask = undef;
    my $chipCount = undef;
    my $expCount = 0;
    my $totalChipCount = 0;
    my $totalMask = 0;
    my $exp_id = undef;

    # get mask for each exposure, and bin
    foreach my $row ( @{$exp_ids} ) {

        $exp_id = @{$row}[0];

        if (!$self->{_gpc1Db}->getMagicMaskStats($exp_id, \$meanMask, \$sumMask, \$chipCount )) {next;}


        if (!$meanMask) {next;}
        #print "expId=$exp_id meanMask=$meanMask sumMask=$sumMask nChips=$chipCount\n";
        $expCount++;
        $totalMask += $sumMask;
        $totalChipCount += $chipCount;

        foreach my $bin (@bins) {
            if ($meanMask <= ($bin+$interval)) {

                $histogram{$bin}++;
                last;
            }
        }
    }

    my $totalMaskFrac;

    if ($totalChipCount > 0) {

        $totalMaskFrac = sprintf( "%.1f", ($totalMask/$totalChipCount) * 100.0);
    }
    else {

        $totalMaskFrac = "NA";
    }

    # write data to GNUPLOT data file
    print "* Found mask values for $expCount exposures out of $totalExp\n";
    open (GNUDAT, ">".$tmpFile->filename);
    my $accum = 0;
    my $maxBin = 0;
    foreach my $bin (@bins) {

        $accum += $histogram{$bin};
        if ($histogram{$bin} > $maxBin) {$maxBin = $histogram{$bin};}
        print GNUDAT "$bin $histogram{$bin} $accum\n";
    }
    close(GNUDAT);


    $maxBin = $maxBin * 1.1;
    $accum = $accum * 1.1;

    my $title = "Magic Mask Fraction for $expCount exposures\\nData taken between '$begin' and '$end'\\nTotal masked fraction is $totalMaskFrac%";

    # make histogram
    open (GP, "|$GNUPLOT -persist") or die "no gnuplot";
    use FileHandle;
    GP->autoflush(1);
    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$histoOutputFile\";";}
    print GP
        "set term $self->{_outputFormat};" .
        "set title \"$title\";" .
        "set grid;" .
        "set boxwidth;" .
        "set style data histogram;" .
        "set yrange [0:$maxBin];" .
        "set style histogram rowstacked;" .
        "set style fill solid border -1;" .
        "set ylabel \"Number of Exposures\";" .
        "set xlabel \"Magic Mask Fraction\";" .
        "set boxwidth 0.75;" .
        "set xtic rotate by -90 scale 0;" .
        "plot '".$tmpFile->filename."' using 2:xtic(1) notitle lt 1;" .
        "\n";

    close GP;

    # make cumulative distribution
    open (GP, "|$GNUPLOT -persist") or die "no gnuplot";
    use FileHandle;
    GP->autoflush(1);
    if ($self->{_outputFormat} ne "X11") {print GP "set output \"$distOutputFile\";";}
    print GP
        "set term $self->{_outputFormat};" .
        "set title \"Cumulative distribution of $title\";" .
        "set yrange [0:$accum];" .
        "set key left top;" .
        "set grid xtics;" .
        "set xlabel \"Magic Mask Fraction\";" .
        "set ylabel \"Number of Exposures\";" .
        "set xtic rotate by -90 scale 0;" .
        "plot " .
        "'".$tmpFile->filename."' using 1:3 notitle with lines lt 2 lw 2\n";

    print GP "\n";
    close GP;

    return 1;
}
1;
