#!/usr/bin/perl

my $debug=0;
my $save_observations=0; # observation saving needs modification to prevent writing
                         # of duplicate observations
                         # observation saving is currently disabled

# NEO confirmation page attributor
#
# Overview
# 1. Pull ephemerides and observation data from MPC NEO confirmation pages
# 2. For each object, find MOPS fields that the object will appear in
# 3. For each field with object intersections, find candidate tracklets
# 4. Report candidate tracklets

# Database Tables
# ephemerides
# observations
# candidate_tracklets

# For each object on the NEO confirmation page at
# http://www.cfa.harvard.edu/iau/NEO/ToConfirm.html
#
# Retrieve the object designations by parsing the NEO confirmation top page
#
# Retrieve the ephemerides data using pull_url()
# Retrieve the observation data using pull_url()

use strict;
use LWP::UserAgent; # pull html data from the web
use DBI;
use LibNEOCP qw(write_fp_files write_obs_files convert_to_mjd connect_database);
use NEOCP::Ephemerides;

my $dbh=connect_database();
my %test_arg;
my $ephem_test=NEOCP::Ephemerides->new($dbh,\%test_arg);

my @neocp_objs; # array of obj labels from neocp
my $neocp_top_url="http://www.minorplanetcenter.org/iau/NEO/ToConfirm.html"; # top page for neo confirmations

my $ephem_url="http://scully.harvard.edu/~cgi/confirmeph"; # url for ephemerides
my $obs_url="http://scully.cfa.harvard.edu/~cgi/ShowObsOrbs.COM?Obj=designation&obs=y"; # url template for observations
my $neocp_html; # html source
my $ephem_html; # html source
my $obs_html; # html source
my $obscode="F51"; # PS1 observatory code

# Use a hash to pass POST parameters
my %ephem_post_param; # objects are selected via checkboxes

my @ephem_data; # array to hold all ephemerides data for an object
my @obs_data; # array to hold all observation data for an object

my %neocp_data; # hash containing parsed data from the NEO CP
# Example
# Ephemerides year for an object is in $neocp_data{ephem}->{object_label}->[idx]->{date}
# Observation year for an object is in $neocp_data{obs}->{object_label}->[idx]->{date}

my $obj_cnt=0;

# Pull HTML data
# ****************************************************************************

# Retrieve current data from the neo cp
$neocp_html=pull_url("top",$neocp_top_url);
parse_top_page($neocp_html);

# For each object label retrieve the ephemerides
print "Will process the following objects\n";

foreach my $obj (@neocp_objs) { # process each obj on the neocp
    $obj_cnt++;
    print "$obj\n";
    %ephem_post_param=('Obj01'=>$obj, # objects are selected via checkboxes
                       'obscode'=>$obscode, # set the observatory code
                       'raty'=>'d', # positions in decimal units
                       'mot'=>'d', # report motion as degrees/day
                       'int'=>'0', # ephemeris interval of one hour
                       'dmot'=>'p' # total motion and direction
                       ); 
    $ephem_html=pull_url("ephem",$ephem_url,%ephem_post_param);

    # Process the ephemerides
    parse_ephemerides($ephem_html,$obj);

    if($save_observations) {
        my $filled_obs_url=$obs_url;
        if($filled_obs_url=~s/designation/$obj/){} # subst object label
        $obs_html=pull_url("obs",$filled_obs_url);
        parse_observations($obs_html,$obj);
    }
    # last; # early break out for testing
}

print "Processed $obj_cnt objects\n";

write_ephemerides(); # write ephem data to the db
if($save_observations) { write_observations(); } # write obs data to the db

# Write data files for FP and mopsdiffcor
foreach my $o (@neocp_objs) {
    write_ephem_as_tracklets($o);
    if($save_observations) { write_obs_files($dbh,$o); }
#    last; # early break for testing
}

exit;

# Find MOPS fields
# ****************************************************************************

# Write a file of MOPS fields
# Write the MPC ephemerides in MOPS tracklet form
# Run fieldProximity to determine field intersections

# Write MOPS fields to be used by fieldProximity
sub write_mops_fields {
}

# Write ephemerides data to track files
sub write_ephem_as_tracklets {
    my ($designation)=@_;
    write_fp_files($dbh,$designation);
}

# Perform fieldProximity on MOPS fields and MPC NEOCP tracks
sub run_fp {
}

# Find candidate tracklets
# ****************************************************************************

# Candidate tracklets are those that are similar to the ephemerides that
# intersect the same MOPS field

sub find_candidate_tracklets {
}

sub write_candidate_tracklets {
}

# Write ephemerides data to the db
# Data is stored as an array of hashes within a hash.
sub write_ephemerides {
    my $insert_cnt=0;
    my $designation;
    
    foreach my $e (@{$neocp_data{ephem}}) {
        my $ephem_existing_ref=retrieve_saved_ephemerides($e->{designation});
        my $hour=int($e->{hour});
        my $compound_date="$e->{year}-$e->{month}-$e->{day} $hour";
        if(defined($ephem_existing_ref->{$compound_date})){
            # dont insert duplicate data
        }
        else {
            $insert_cnt++;
            # iterate over all the ephemerides records
            my $sql="INSERT INTO ephemerides (timestamp, designation, year, month, day, hour, mjd_datetime, ra, `dec`,
                                              elong, mag_v, deg_per_day, pa, azimuth, altitude)
                                              VALUES
                                              (NOW(),?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
            my $sth=$dbh->prepare($sql);
            my $result=$sth->execute($e->{designation},
                                     $e->{year},
                                     $e->{month},
                                     $e->{day},
                                     $e->{hour},
                                     $e->{mjd_datetime},
                                     $e->{ra},
                                     $e->{dec},
                                     $e->{elong},
                                     $e->{mag_v},
                                     $e->{deg_per_day},
                                     $e->{pa},
                                     $e->{azimuth},
                                     $e->{altitude});
        }
    } # end foreach
    print STDERR "Inserted $insert_cnt records.\n";
} # end sub

# Write observation data to the db
# Pass data as an array of hashes.
sub write_observations {

    foreach my $o (@{$neocp_data{obs}}) {
        
        my $sql="INSERT INTO observations (designation, year, month, day, hour, ra_hr, ra_min, ra_sec,
                                           dec_deg, dec_min, dec_sec, magnitude)
                 VALUES
                 (?,?,?,?,?,?,?,?,?,?,?,$o->{magnitude})";
        my $sth=$dbh->prepare($sql);

        my $result=$sth->execute($o->{designation},
                                 $o->{year},
                                 $o->{month},
                                 $o->{day},
                                 $o->{hour},
                                 $o->{ra_hr},
                                 $o->{ra_min},
                                 $o->{ra_sec},
                                 $o->{dec_deg},
                                 $o->{dec_min},
                                 $o->{dec_sec});                                 
    }
}

# From the ephemerides HTML, make a hash for each record and push it onto the
# ephemerides data array
sub parse_ephemerides_notused {
    my $mpc_desig; # mpc designation
    my $date;
    my $ra;
    my $dec;
    my $elong;
    my $mag_v;
    my $deg_per_min;
    my $pa; # position angle

    # For each ephemerides record, extract the data and push it to the ephem array
    my %ephem;
    $ephem{designation}=$mpc_desig;
    $ephem{date}=$date;
    $ephem{ra}=$ra;
    $ephem{dec}=$dec;
    $ephem{elong}=$elong;
    $ephem{mag_v}=$mag_v;
    $ephem{deg_per_min}=$deg_per_min;
    $ephem{pa}=$pa;

    push(@ephem_data,\%ephem);
}

# Retrieve HTML data from a URL
# Performs two types of retrievals (ephem or obs)
# Return the HTML content
#
# $type ephem or obs
# $url URL of the NEOCP
# %param hash of parameters needed to retrieve correct data
sub pull_url {
    my ($type,$url,%param)=@_;

    # User agent object
    use LWP::UserAgent;
    my $ua = LWP::UserAgent->new;
    $ua->agent("Perl/5.10.1 ");
    if($type eq "obs") {
        # Spoof the referer to get the observations
        $ua->default_header('Referer', 'http://scully.harvard.edu/~cgi/confirmeph');
    }

    # Request
    my $req;

    if($type eq "obs") { # Observations are a GET request
        $req = HTTP::Request->new(GET => $url);
        $req->content_type('application/x-www-form-urlencoded');
    }

    if($type eq "top") { # Top page is a GET request
        $req = HTTP::Request->new(GET => $url);
        $req->content_type('application/x-www-form-urlencoded');
    }

    # Pass request to user agent
    my $res; # response
    if($type eq "obs" || $type eq "top") {
        $res = $ua->request($req); #response
    }
    elsif ($type eq "ephem") { # Ephemerides are a POST request
        $res = $ua->post($url,\%param);
    }
    else { return 0; } # no type is valid

    # Check response
    if ($res->is_success) {
        return $res->content;
    }
    else { # an error has occurred
        print $res->status_line, "\n";
    }
}

# Retrieve the current NEOCP objects.
#
# Example of what is being parsed
# This is the comments section of the NEOCP
#
# <p>Select object(s) from the current list of objects needing confirmation
# (discovery date, rough current position and magnitude given):
# <br><input type="checkbox" name="Obj01" VALUE="BQ23079" CHECKED> BQ23079<sub>   </sub> [2009 Sept. 21.4 UT.   R.A. = 07 06.9, Decl. = +23 55, V = 17.2] Added Sept. 22.05 UT <b>[1 nighter]</b>
# <br><input type="checkbox" name="Obj02" VALUE="9S24D95" CHECKED> 9S24D95<sub>   </sub> [2009 Sept. 21.3 UT.   R.A. = 01 26.3, Decl. = +17 04, V = 19.2] Updated Sept. 21.98 UT
# <br><input type="checkbox" name="Obj03" VALUE="9SAF070" CHECKED> 9SAF070<sub>   </sub> [2009 Sept. 21.3 UT.   R.A. = 02 51.5, Decl. = +20 46, V = 20.5] Updated Sept. 21.87 UT <b>[1 nighter]</b>
# <p><hr><p>Select your viewing point:
sub parse_top_page {
    my ($html)=@_;
    my @data=split(/\n/,$html);
    my $get_labels=0;
    foreach my $line (@data) {
        if($get_labels){
            if($line=~/VALUE=\"(\w+)\"/){ # extract label from html
                push(@neocp_objs,$1);
            }
        }
        if($line=~/^\(discovery date\, rough current position and magnitude given\)\:/){
            $get_labels=1; # start retrieval on next line
        }
        if($line=~/^\<p\>\<hr\>\<p\>Select your viewing point:/){ # end retrieval
            $get_labels=0;
        }
    }
}

# Extract ephemerides data from HTML
#
# Source data example
# Date       UT      R.A. (J2000) Decl.  Elong.  V        Motion     Object     Sun         Moon        Uncertainty
#             h                                        °/day   P.A.  Azi. Alt.  Alt.  Phase Dist. Alt.
# 2009 10 28 22    09.7953    +13.010     70.7  17.9    0.51  095.6  094  +28   +56    0.76  167  -36   Map/Offsets
# 2009 10 28 23    09.7968    +13.008     70.8  17.9    0.51  095.5  099  +14   +54    0.77  167  -22   Map/Offsets
# 2009 10 29 00    09.7982    +13.006     70.8  17.9    0.51  095.5  104  +01   +46    0.77  166  -09   Map/Offsets
sub parse_ephemerides {
    my ($html,$obj_label)=@_;
    
    my $designation=$obj_label;
    my $year; # calendar date value
    my $month; # calendar date value
    my $day; # calendar date value
    my $hour; # calendar date value
    my $mjd_datetime; # mjd converted date time
    my $ra_hour; # deprecated
    my $dec_deg; # deprecated
    my $ra_minsec; # deprecated
    my $dec_minsec; # deprecated
    my $ra; # decimal units
    my $dec; # decimal units
    my $elong; # elongation
    my $mag_v; # magnitude
    my $deg_per_sec; # deprecated
    my $deg_per_min;
    my $deg_per_day; # velocity corresponding to MOPS tracklet velocity
    my $pa; # position angle
    my $azimuth;
    my $altitude;
    
    my @data=split(/\n/,$html);
    my $get_header=0;
    my $header_cnt=0;
    
    # the header checks here are used to skip the header text in the source data
    foreach my $line (@data) {
        if($line=~/^\s+\<pre\>/){
            $get_header=1;
        }
        
        if($get_header){
            $header_cnt++;
        }
        
        if($header_cnt>=3){
            if($line=~/\s+\<\/pre\>/){
                $header_cnt=0;
                $get_header=0;
            }
        }
        
        if($header_cnt > 3) {
            my @data=split(/\s+/,$line);
            
            # data is retrieved in the order that it appears
            $year=$data[0];
            $month=$data[1];
            $day=$data[2];
            $hour=$data[3];
            $ra=$data[4];
            $dec=$data[5];
            $elong=$data[6];
            $mag_v=$data[7];
            $deg_per_day=$data[8];
            $pa=$data[9];
            $azimuth=$data[10]; # not used for processing
            $altitude=$data[11]; # not used for processing
                          
            my %eph=(); # new hash to hold record data
            $eph{designation}=$designation;
            $eph{year}=$year;
            $eph{month}=$month;
            $eph{day}=$day;
            $eph{hour}=$hour;
            $eph{mjd_datetime}=convert_to_mjd($day,$month,$year,$hour);
            $eph{ra}=$ra;
            $eph{dec}=$dec;
            $eph{elong}=$elong;
            $eph{mag_v}=$mag_v;
            $eph{deg_per_day}=$deg_per_day;
            $eph{pa}=$pa;
            $eph{azimuth}=$azimuth;
            $eph{altitude}=$altitude;
            
            push(@{$neocp_data{ephem}},\%eph); # load data into global data structure
        }
    }   
}

# Extract observation data
# $html HTML source data
# $obj_label Object label of object's observations
sub parse_observations {
    my ($html,$obj_label)=@_;

    my $year;
    my $month;
    my $day;
    my $hour;
    my $ra_hr;
    my $ra_min;
    my $ra_sec;
    my $dec_deg;    
    my $dec_min;
    my $dec_sec;
    my $magnitude;  
    my $designation;

    my @data=split(/\n/,$html);
    my $get_data=0; # binary flag to start parsing data
    
    foreach my $line (@data) {

        if($line=~/REMARK\:/){
            $get_data=0; # stop getting data     
        }
        
        if($get_data){
            if($line=~s/^\s+//) {} # Remove leading space
            if($line=~s/\s+$//) {} # Remove trailing space

            my @content=split(/\s+/,$line);

            $designation=$content[0];
            if($content[1]=~/(\d\d\d\d)$/){ $year=$1; }
            $month=$content[2];
            $day=$content[3];
            $ra_hr=$content[4];
            $ra_min=$content[5];
            $ra_sec=$content[6];
            $dec_deg=$content[7];
            $dec_min=$content[8];
            $dec_sec=$content[9];
            $magnitude=$content[10];

            if($debug){                          
                print "$designation $year $month $day $hour $ra_hr $ra_min $ra_sec $dec_deg $dec_min $dec_sec $magnitude\n";
            }
            
            my %obs=();
            $obs{designation}=$designation;
            $obs{year}=$year;
            $obs{month}=$month;
            $obs{day}=$day;
            $obs{hour}=$hour;
            $obs{ra_hr}=$ra_hr;
            $obs{ra_min}=$ra_min;
            $obs{ra_sec}=$ra_sec;
            $obs{dec_deg}=$dec_deg;
            $obs{dec_min}=$dec_min;
            $obs{dec_sec}=$dec_sec;

            if($magnitude=~/[A-Z]/) { $magnitude='NULL'; } # this needs to be set to null instead of -1
            $obs{magnitude}=$magnitude;
            
            # push hash ref onto array contained by master data hash
            push(@{$neocp_data{obs}},\%obs);
        }
        
        if($line=~/\<pre\>/){
            $get_data=1; # start getting data     
        }
        
    }
}

# Retrieve the ephemerides data for an existing designation
sub retrieve_saved_ephemerides {
    my($designation)=@_;
    my $sql="SELECT * FROM ephemerides WHERE designation=?";
    my $sth=$dbh->prepare($sql);
    $sth->execute($designation);
    my $ref;
    my %ephem_existing;
    my $compound_date;
    while($ref=$sth->fetchrow_hashref()) {
        my $hour=int($ref->{hour});
        $compound_date="$ref->{year}-$ref->{month}-$ref->{day} $hour";
        $ephem_existing{$compound_date}=1;
    }
    return \%ephem_existing;
}
