#!/usr/bin/env perl

# This program was created to facilitate processing of MPC data. The
# observations file was too large to be conveniently processed. Therefore,
# this program splits that file into manageable pieces. It maintains groups of
# data records where a record consists of one line in a text file. The
# grouping is decided by the first data column.
#
# Usage example
# $ cat mpn.arc | ./keep-together-split.pl
#
# Outputs data to files with the form
# data.out.nnn.mpc
# where nnn is an integer.
#
# Notes
# Given a data file with one record per line: Load each record into memory.
# Write records to disk while ensuring that groups of data stay together.
#
# Example:
#
# field1    field2    field3
#      A        12        11
#      A         8         5
#      A         4         3
#      A        53         1
#      B         3         2
#      B         1        10
#      C         0         1
#
# Splitting this into two files would keep all those records with field1=A
# together in one new file. All the records with field1=B would also be kept
# together in one file. In other words, data groups (indicated by a single
# field), will not be split across files.

# Target lines per file.
my $target_cnt=100000;

my $cnt=0;
my @saved_data=();
my $last_number;
my $current_number;
our $FILE_CNT=1;
my $last_line;
while($line=<STDIN>){
    next if ($line=~/^(\s)*$/); # Ignore blank lines.
    my $number=substr($line,0,5);
    $current_number=$number;

    # This misses the last file.
    if($cnt>$target_cnt && $last_number ne $current_number){
        write_out_data(\@saved_data);
        @saved_data=(); # Clear the array.
        $FILE_CNT++;
        $cnt=0;
    }
    # Save data lines.
    push(@saved_data,$line);
    $last_line=$line;
    $last_number=$current_number;
    $cnt++;
}

# Write the remaining data in the buffer for the last file.
if(@saved_data[0]){
    write_out_data(\@saved_data);
}

sub write_out_data {
    my($d)=@_;
    open(FILE,">data.out.$FILE_CNT.mpc");
    foreach my $l (@{$d}){
        print FILE $l;
    }
    close(FILE);
}
