#!/usr/bin/perl
#
# t2t2
# IHS May 30, 2009: Check for FUTs in Princeton-format TOAs
# and convert to MJDs.  Leave commented-out toas with "C"
# rather than "I" as there are parsing issues if the input "C"
# replaces a column, and then tempo2 dies on trying to read an
# "ignored" TOA -- not worth fixing today!
# Also allow user to replace a site number/letter with new-style
# longer codes.
#
# IHS July 4, 2008: minor bug fixes: prints out overridden ID now
# even if there was a usable ID to begin with.  Also correctly
# prints out requested extra flags in the absence of JUMPs
#
# IHS, March 6, 2007, borrowing from various sources including
# several DJN scripts.
#
# Converts tempo1-format toa files to tempo2 format.
# Should be able to handle Parkes, Jodrell, ITOA and Princeton
# format toas, including cases where not all columns on
# Princeton-format toas are used.
#
# As per email from George Hobbs, March 6 2007:
# 
# A tempo2 file must have as the first line:
# 
# FORMAT 1
# 
# Then you can use a "C" or "#" at the start of a line for a comment. Each 
# real observation line must have
# 
# ID FREQ(MHZ) SAT TOAERR(us) TEL_ID
# 
# Where "ID" is an identifier (e.g. pulsar name, filename ...) that does not 
# contain any white spaces.  Each parameter can be as long or as short as 
# you like and you can leave as many spaces between each parameter.
# 
# Example:
# 
# FORMAT 1
# 1853+13  1402.000 53370.6646017438444     1.48 3 -f asp
# 1853+13  1402.000 53393.5956318214029     0.88 3 -f asp
# 
# On the end of the line you can add as many flags as you like.  Flags start 
# with a "-" sign, but can be as long as you like e.g.
# 
# -inst asp -obs ingrid -tel arecibo
#
# In your parameter file you can then add jumps using (e.g.):
# 
# JUMP -f asp
# JUMP -tel arecibo
# 
# end George email
#
#
# This program will take in a toa file (with toas, not with a list of
# INCLUDEd files) and rearrange the columns to tempo2 format.  
# You may add as many flags as you like on the command line; these
# will get added to the toa lines as flag listings.
# This program will automatically add -info flags with the current
# info number (assuming it encounters many INFO lines in the toa file)
# UNLESS the INFO is overridden on the command line, in which
# case ALL output toa lines will get that info flag, disregarding whatever
# INFO lines may actually be in the toa file.
# When JUMPs are encountered in the file, an extra `jump` flag
# will be appended with an incrementing jump number.
# If there are no JUMPs in the toa file, no jump flags will be written out.
# When commented-out or SKIPped toas are encountered, the toa line(s)
# will be printed out with an initial `I` to be ignored by tempo2.

$usage = 
  "Use: t2t2 <input toa file> <output toa file> [options]\n".
  "Options:\n".
  "  -flag flagname flagvalue (where flagname is not `info` or `jump`; multiple flags allowed)\n".
  "  -id some_id (all ID values will be some_id)\n".
  "  -tel_id telescope-ID (replace all site info with this string)\n".
  "  -info number (to override all info lines in the toa file)\n".
  "  -nodmo (do not write out DM corrections, even if present. Must use this with PRESTO TOAs.)\n".
  "  -type pton/jb/pks/itoa (defaults to assuming pton)\n";

$ttype = 0;
@ttstring = ("pton", "jb", "pks", "itoa");
$nflags = 0;
$i = 0;
$ip = -1;
$forceinfo = 0;
$setid = 0;
$settelid = 0;
$newtelid = "random_place";
$idstring = "uninformative";
$njump = 0;
$useinfo = 0;
$nodmo = 0;

if($#ARGV<1) {
  die $usage;
}
while($i <= $#ARGV) {
#  printf "%d %s\n", $i, $ARGV[$i];
  if ($ARGV[$i] =~ /^-/) {
    if ($ARGV[$i] =~ /^-flag/) {
      $flagname[$nflags] = $ARGV[$i+1];
      $flagval[$nflags] = $ARGV[$i+2];
      if($flagname[$nflags] =~ /^info$/ || $flagname[$nflags] =~ /^jump$/) {
        die $usage;
      }
      $nflags++;
      $i+=3;
    }
    elsif ($ARGV[$i] =~ /^-id/) {                          
      $setid = 1;  
      $idstring = $ARGV[$i+1];
      $i+=2;
    }
    elsif ($ARGV[$i] =~ /^-tel_id/) {                          
      $settelid = 1;  
      $newtelid = $ARGV[$i+1];
      $i+=2;
    }
    elsif ($ARGV[$i] =~ /^-info/) {                          
      $forceinfo = 1;  
      $useinfo = 1;
      $infoval = $ARGV[$i+1];
#      printf "Forcing info to be %d\n", $infoval;
      $i+=2;
    }
    elsif ($ARGV[$i] =~ /^-nodmo/) {                          
      $nodmo = 1;  
      $i+=1;
    }
    elsif ($ARGV[$i] =~ /^-type/) {
      if($ARGV[$i+1] =~ /^pton/) {
        $ttype = 0;
      }
      if($ARGV[$i+1] =~ /^jb/) {
        $ttype = 1;
      }
      if($ARGV[$i+1] =~ /^pks/) {
        $ttype = 2;
      }
      if($ARGV[$i+1] =~ /^itoa/) {
        $ttype = 3;
      }
      $i+=2;
    } else {
      die $usage;
    }
  }
  else { 
    $ip++;
    $infile = $ARGV[$i] if ($ip == 0);
    $outfile = $ARGV[$i] if ($ip == 1);
    if ($ip>1) {
      die $usage;
    }
    $i++;
#    printf "%d %d %s %s\n",$i, $ip,$infile,$outfile;
  }
}

printf "Will convert file %s to file %s, assuming %s format.\n", $infile, $outfile, $ttstring[$ttype];
printf "Forcing all info flags to %d.\n", $infoval if ($forceinfo == 1);
printf "Setting all ID values to %s.\n", $idstring if ($setid == 1);
printf "Setting telescope string to %s.\n", $newtelid if ($settelid == 1);
if ($nflags > 0) {
  printf "Appending the following flags:\n";
  for ($iflag=0; $iflag<$nflags;$iflag++) {
    printf " -%s %s",$flagname[$iflag],$flagval[$iflag]
  }
  printf "\n";
}

open (OUT, ">$outfile") || die "Cannot open $outfile for writing: $!";
open (IN, "$infile") || die "Cannot open $infile for reading: $!";

printf OUT "FORMAT 1\n\n";

$ignoreline = 0;
$jumppair = 0;

while(<IN>) {
  chomp;
  $len = length($_);
  $col1 = substr($_,0,1);
#  printf "%s has length %3d and first column %s\n",$_,$len,$col1;
  $id = sprintf "nothingyet";
  $freq = -200.;
  $toa = 1.0;
  $toaerr = 1.0;
  $site = "Q";
  $dmo = 0.;
  $istoa = 0;
  $toaline = 0;
  $singlebad = 0;
  if($len > 1) {  #  Don't deal with blank lines
#    printf "%s\n%2d: Toa, istoa, ignore, singlebad,jumppair: %1d %1d %1d %1d %1d\n",$_,1,$toaline,$istoa,$ignoreline,$singlebad,$jumppair;
    if($col1 =~ /C/) {  # Either a comment line or a commented-out TOA
      $rest = substr($_,0,$len);
      $lenr = $len;
      while ($istoa == 0 && $lenr > 30) { # See if the rest of the line
                            # should be interpreted as a commented-out TOA
        $rest = substr($rest,1,$lenr);
        $lenr = $lenr-1;
        $istoa = &toaparse($rest);
        if($istoa == 1)  {  # To be written out later, with flags and `I`
          $toaline = 1;
          $singlebad = 1;
#          printf "I think this is a commented-out TOA:\n %s %12.5f %s %s %1s\nValues of toaline, singlebad and ignoreline are: %d %d %d\n", $id,$freq,$toa,$toaerr,$site,$toaline,$singlebad,$ignoreline;
 #         break;
        }
      }
#    printf "%s\n%2d: Toa, istoa, ignore, singlebad,jumppair: %1d %1d %1d %1d %1d\n",$_,2,$toaline,$istoa,$ignoreline,$singlebad,$jumppair;
      if($istoa == 0) { # Looks like a comment -- just write it out
        printf OUT "%s\n", $_; 
        $toaline = 0;
        $singlebad = 0;
      }
#    printf "%s\n%2d: Toa, istoa, ignore, singlebad,jumppair: %1d %1d %1d %1d %1d\n",$_,3,$toaline,$istoa,$ignoreline,$singlebad,$jumppair;
    }
    else { # Check to see if we have a TOA command; treat as needed
      ($firstword) = split;
#      printf "Trying to figure out what to do with %s\n",$firstword;
      if($firstword =~ /^SKIP$/) {
        $toaline = 0;
        $ignoreline = 1;
      }
      elsif($firstword =~ /^NOSKIP$/) {
        $toaline = 0;
        $ignoreline = 0;
      }
      elsif($firstword =~ /^MODE$/) {
        $toaline = 0;
      }
      elsif($firstword =~ /^INFO$/) {
        $toaline = 0;
        $useinfo = 1;
#        printf "Caught an INFO flag, value ";
        if($forceinfo == 0) {
          ($jnq,$infoval) = split;
#          printf "%d",$infoval;
        }
#        printf "\n";
      }
      elsif($firstword =~ /^JUMP$/) {
        $toaline = 0;
        if($jumppair == 0) {
          $njump++;
          if($njump == 1) {
            $flagname[$nflags] = "jump";
            $nflags++;
          }
          $flagval[$nflags-1] = $njump -1;
        }
        $jumppair = 1-$jumppair;
      }
      elsif($firstword =~ /^[A-Z]+/ || $firstword =~ /PHA[1-2]/) { 
  			# some other command, not treated specially
        $toaline = 0;
        printf OUT "%s\n", $_; 
#    printf "%s\n%2d: Toa, istoa, ignore, singlebad,jumppair: %1d %1d %1d %1d %1d\n",$_,4,$toaline,$istoa,$ignoreline,$singlebad,$jumppair;
      }
      else { # all other possibilities exhausted -- must be a TOA line
        $toaline = 1;
        $success = &toaparse($_);
        if($success == 0 && $len > 30) {
#          printf "%s %12.5f %s %s %1s\n", $id,$freq,$toa,$toaerr,$site;
          die "Bad toa line: $_\nTry another TOA format option.";
        }
#    printf "%s\n%2d: Toa, istoa, ignore, singlebad,jumppair: %1d %1d %1d %1d %1d\n",$_,5,$toaline,$istoa,$ignoreline,$singlebad,$jumppair;
      }
    }
#      printf "Time to decide what to do:\n %s %12.5f %s %s %1s\nValues of toaline, singlebad and ignoreline are: %d %d %d\n", $id,$freq,$toa,$toaerr,$site,$toaline,$singlebad,$ignoreline;
      if($toaline == 1) {
        if($ignoreline == 1 || $singlebad == 1) {
          printf OUT "C ";
          if($singlebad == 1) {
            $singlebad = 0;
          }
        }
#        if($id =~ /\s*/) {
        if(!($id =~ /[a-zA-Z1-9]/)) {  # ID contains no letters or numbers
          if($setid == 0) {
             die "Can't find usable ID value.  Rerun with -id option.\n";
          }
#          else {
#            $id = sprintf "%s", $idstring;
#          } 
        }
        if($setid == 1) {
          $id = sprintf "%s", $idstring;
        }
        if($settelid == 1) {
          $site = sprintf "%s", $newtelid;
        }
        printf OUT "%s %12.5f %s %s %s", $id,$freq,$toa,$toaerr,$site;
        if($dmo != 0. && $nodmo == 0) {
          printf OUT " -dmo %e", $dmo;
        }
        if($useinfo == 1) {
          printf OUT " -info %1d", $infoval;
        }
        if($jumppair==1) {
          for($iflag=0;$iflag<$nflags;$iflag++) {
            printf OUT " -%s %s",$flagname[$iflag],$flagval[$iflag];
          }
        }
        if($jumppair==0) {
          if($njump > 0) {
            for($iflag=0;$iflag<$nflags-1;$iflag++) {
              printf OUT " -%s %s",$flagname[$iflag],$flagval[$iflag];
            }
          }
          else {
            for($iflag=0;$iflag<$nflags;$iflag++) {
              printf OUT " -%s %s",$flagname[$iflag],$flagval[$iflag];
            }
          }
        }
        printf OUT "\n";
      }
  }
}

sub toaparse {
  my ($line) = ($_[0]);  # Use a local variable for the line to be parsed


  if($ttype == 0) { # Princeton format
    
# From the tempo reference page: 
# Princeton Format
# 
#   columns     item
#     1-1     Observatory (one-character code)
#     2-2     must be blank
#     16-24   Observing frequency (MHz)
#     25-44   TOA (decimal point must be in column 30 or column 31)
#     45-53   TOA uncertainty 
#     69-78   DM correction (pc cm^-3)

    $site = substr($line, 0,1);
    $freq = substr($line, 15,9);
    $toa = substr($line, 24,20);
    $toaerr = substr($line, 44,9);
    $dmo = substr($line, 68,10);
    $id = substr($line, 6,8);  # This will often, but not always, be the case
    $nscan = substr($line, 1,5);  # This will often, but not always, be the case
 
# Parse TOA itself to check for FUT and deal with that if necessary
    if(substr($toa,5,1) =~/\./ ) {
      $mjd = substr($toa,0,5);
      $fracmjd = substr($toa,6,14);
      #printf "split type-1 toa into %s and %s\n",$mjd,$fracmjd
    }
    elsif(substr($toa,6,1) =~/./ ) {
      $mjd = substr($toa,0,6);
      $fracmjd = substr($toa,7,13);
    }
    if($mjd < 30000) {
      $mjd += 39126;
      $toa = sprintf "%5d.%14s", $mjd,$fracmjd;
    } 

    if(($site =~ /[0-9]/ || $site =~ /[a-z]/ || $site =~ /@/) && $freq > 10.) {
      return 1;
    }
    else {
      return 0;
    } 
  }

  if($ttype == 1) { # JBO format -- essentially the same as Parkes, but
                    # the ID is elsewhere.
    
# From the tempo reference page: 
# Parkes Format
# 
#   columns     item
#     1-1     must be blank
#     26-34   Observing frequency (MHz)
#     35-55   TOA (decimal point must be in column 42)
#     56-63   Phase offset (fraction of P0, added to TOA)
#     64-71   TOA uncertainty
#     80-80   Observatory (one-character code)
 
    $colcheck = substr($line,0,1);
    $site = substr($line, 79,1);
    $freq = substr($line, 25,9);
    $toa = substr($line, 34,21);
    $toaerr = substr($line, 63,8);
    $id = substr($line, 17,8);  # This will often, but not always, be the case

    if($colcheck !~ " ") {  # Wrong format
      return 0;
    }
    if(($site =~ /[0-9]/ || $site =~ /[a-z]/ || $site =~ /@/) && $freq > 10.) {
      return 1;
    }
    else {
      return 0;
    } 
  }

  if($ttype == 2) { # Parkes format
    
# From the tempo reference page: 
# Parkes Format
# 
#   columns     item
#     1-1     must be blank
#     26-34   Observing frequency (MHz)
#     35-55   TOA (decimal point must be in column 42)
#     56-63   Phase offset (fraction of P0, added to TOA)
#     64-71   TOA uncertainty
#     80-80   Observatory (one-character code)
 
    $colcheck = substr($line,0,1);
    $site = substr($line, 79,1);
    $freq = substr($line, 25,9);
    $toa = substr($line, 34,21);
    $toaerr = substr($line, 63,8);
    $id = substr($line, 1,18);  # This will often, but not always, be the case

    if($colcheck !~ " ") {  # Wrong format
      return 0;
    }
    if(($site =~ /[0-9]/ || $site =~ /[a-z]/ || $site =~ /@/) && $freq > 10.) {
      return 1;
    }
    else {
      return 0;
    } 
  }

  if($ttype == 3) { # ITOA format

# From the tempo reference page: 
# ITOA Format
# 
#   columns     item
#     1-2     ignored, but must not be blank (often PSRNAME goes here)
#    10-28    TOA (decimal point must be in column 15)
#    29-34    TOA uncertainty 
#    35-45    Observing frequency (MHz)
#    46-55    DM correction (pc cm^-3)
#    58-59    Observatory (two-letter code)
 
    $colcheck = substr($line,0,1);
    $sitein = substr($line, 57,2);
    $freq = substr($line, 34,11);
    $toa = substr($line, 9,19);
    $toaerr = substr($line, 28,6);
    $dmo = substr($line, 45,10);
    $id = substr($line, 0,9);  # This will often, but not always, be the case

    if($colcheck =~ " ") {  # Wrong format
      return 0;
    }
    if(($site =~ /[A-Z][A-Z]/) && $freq > 10.) {
      $site = sprintf "%sUTC", $sitein;
      return 1;
    }
    else {
      return 0;
    } 
  }
}