#!/usr/local/bin/perl -w 

### $Id$
### Author:
### NAME: genMetadata.pl
###
### This software is Copyright (C) 2008,
### ACE Science Center, SRL, California Institute of Technology.
### All rights reserved. Unauthorized reproduction prohibited.
###
### The software is provided to you without charge,
### and with no warranty. You may give away copies of
### this software, including sources, provided that
### this notice is included in all the files.
###
###
### CALLING SEQUENCE:
###	genMetadata.pl /full/path/Hfile.h  /full/path/XMLfile.xml
###
###
### PURPOSE:
###	O/P a metadata file based upon info from
###	a (dot)H file and a (dot)XML file (given on the cmdline).
###
###	The format is:
###	Line 1: nrecords (entries)
###	LInes 2 thru nrecords+1: parameter_key followed by format_string,
###	space- or tab-delimited (does not matter...)
###
### INPUT:
###     Cmdline: two file name strings:
###		 [fullpath/]filename.h
###		 [fullpath/]filename.xml
###     Example:
###  genMetadata4HDFreader.pl /users/asc/level2/include/swepam_level2_data_1hr.h
###                 /users/asc/level2/HDFReaders/devel/XML/NumericalData-ACE-SWEPAM-HDF_PT64S.xml
###
### OUTPUT:
###	STDOUT: normal O/P to STDOUT
###	ERR Diagnostics    to STDOUT
###
### RETURN:
###
###
### MODIFICATION HISTORY:
### =====================
### Date: Thu, 26 Jun 2008 22:37:51 -0700
### To: glennh@srl.caltech.edu
### From: Andrew Davis <ad@srl.caltech.edu>
### Subject: Task to keep you occupied
###
### Hi Glenn,
### While I'm off having fun, here is a task to keep you busy:
###
###    In /users/asc/level2/HDFReaders/metadata you'll find two files:
###            swepam_level2_data_1hr.txt
###            swepam_level2_data_64sec.txt
###
### These are metadata files used by the read_hdf_driver program.
### (See init_metadata.c in /users/asc/level2/HDFReaders)
###
### Currently, the files are just stubs, containing only a few of the entries
###    that really need to be in the files.
###
### The format is:
### Line 1: nrecords (entries)
### LInes 2 thru nrecords+1: parameter_key followed by format_string,
### space- or tab-delimited (does not matter...)
###
### Please write a perl script that will generate one of these files
### using a .h file
### from /users/asc/level2/include as input (swepam_level2_data_1hr.h,
### for example).
###
### The .h file will give you the parameter_keys, but not the format_strings.
### I will have to think of a way to tell your perl script about the
### correct format_strings.
### For now, just output some place_holder format_string.
###
### That should do until Tuesday morning, when I'll be back to work.
###
### All the best,
### Andrew
###
### ================
### From:   ad@srl.caltech.edu
### Subject:    Re: Using XML::Simple
### Date:   July 2, 2008 3:55:20 PM PDT
### To:     glennh@srl.caltech.edu
###
### Hi Glenn,
### OK, here is something to make this task meaty again.
###
### Your program must read BOTH the XML file, AND
### the appropriate .h file.
###
### From the XML file, you get a list of parameter_keys and
###     format_strings.
### From the .h file, you get just a list of parameter_keys.
###
### The keys in the XML file can be a subset of the keys
### in the .h file, that's OK.
### Also, the keys in the XML file may be out of order with
### the keys in the .h file, that's also OK.
###
### However, your program should check that each key found
### the XML file has a match in the .h file.
### If not, that's an error.
###
### If the above test is passed, then output the metadata
### as usual,
### EXCEPT:
###        Add a 3rd column, containing an index value.
###
### For each record in the output, the index value shall
### be the index into the parameter_key list obtained
### from the .h file, counting from 0.
###
### Call me if this is not clear!
### Andrew
### ================
###
###
### MODIFICATION HISTORY:
### =====================
### 2008Jul15-GRH-redefined the cmdline parameters to require-
###	Full path-filename strings: path/filename.h & path/filename.xml
###	Fixed O/P diagnostic msg in chk for missing (dot)H line(s) to
###	report all missing param.names; not just last one found.
### 2008Jun27- Created.
###
###################################################

# You MUST remember THIS:  (not the song!!)
#	FALSE: in PERL is 0(zero) or "" or null
#	TRUE : is anything that is not FALSE
#
# RETURN VALUES in this program:
#   0(zero) is a good or OK return condition.
#   Anything other than 0 is a problem indicator.

use strict;
use Cwd;
use XML::Simple;

### Initialization

my $dbug = 0;    # 0=Production

our $nargs     = 1;     # of args(less one) expect.on cmdline
our $dotH_fn   = "";    #  (dotH) filename
our $dotX_fn   = "";    #  (dotX) filename
our @HdataType = ();
our $Hcount    = 0;

our $Description = "";
our $InfoURL = "";
our $PolicyURL = "";
our $Ack = "";

our %DF         = ();    # Hash of parm.names (keys) & data format strings
our %VMin       = ();    # Hash of parm.names (keys) & valid_min
our %VMax       = ();    # Hash of parm.names (keys) & valid_max
our %Name       = ();    # Hash of parm.names (keys) & Names
our %Units      = ();    # Hash of parm.names (keys) & Units
our %Fill       = ();    # Hash of parm.names (keys) & Fill-values
our %SCtype     = ();    # Hash of parm.names (keys) & Scaletypes (Log/Lin)
our %Dim        = ();    # Hash of parm.names (keys) & Dimensions (array sizes)
our %H          = ();    # Hash of parm.names (keys) & index into (dot)H file
our %Description    = ();    # Hash of parm.names (keys) & parameter descriptions
our %Legends    = ();    # Hash of parm.names (keys) & Legends

my %p = (               # init_param_hash_struct
	dbug => $dbug,      # scalar

	#	periods => [qw( 12m/ 1d/ 1h/ 2h/ )],    # array
	#	dir     => "",                          # string
	#	msg     => "",                          # string
	#	DOY_tbl => [ () ],                      # defines & inits arrays
);
my $rtnval = 0;

# CHECK CMDLINE
exit(-1) if ( chkCmdline() );

$dotH_fn = $ARGV[0];    # Filenames fm cmdline
$dotX_fn = $ARGV[1];

# Ensure .h suffix
if ( $dotH_fn !~ /.h$/ ) {
	$dotH_fn =~ s/$/.h/;
}

# READ DotH FILE
# Gets DotH filename from ARGV[0] &
#   Gets DotH lines & count of physical param lines
if ( readin_Hfile() != 0 ) {
	exit(-1);
}

if ($dbug) {
	printf STDERR "H:\n";
	while ( my ( $k, $v ) = each %H ) {
		printf STDERR "%s = %d\n", $k, $v;
	}
}

# READ XML FILE
if ( readin_Xfile() != 0 ) {
	exit(-1);
}

# CHECK EACH KEY IN (dot)XML FILE HAS A MATCH IN (dot)H FILE
if ( chkXMLmatchInDotH() != 0 ) {
	exit(-1);
}

#my @xkeys = keys(%DF);
#my @hkeys = keys(%H);
#$p{msg}   = "";

#XKEY:
#foreach my $xkey (@xkeys) {
#  foreach my $hkey (@hkeys) {
#    next XKEY if ( $hkey eq $xkey );
#  }

#  # Whoa!! No match found in the (dot)H file!
#  if(length($p{msg}) > 0) {
#    # start w/tab if after 1st err msg
#    $p{msg} .= "\t";
#  }
#  # Accum err diag msgs
#  $p{msg} .= "(dot)H file missing \'$xkey\'\n";
#  $rtnval = -1;
#}

#if ( $rtnval == -1 ) {
#  printmsg();
#  exit(-1);
#}

# WRITE O/P
if ( print2op() ) {
	exit(-1);
}
exit(0);
###------  end  main  -------------------------

###------  start  chkXMLmatchInDotH  -----------------------
sub chkXMLmatchInDotH {

	# CHECK EACH KEY IN (dot)XML FILE HAS A MATCH IN (dot)H FILE
	my @xkeys  = keys(%DF);
	my @hkeys  = keys(%H);
	my $errflg = 0;
	$p{msg} = "";

  XKEY:
	foreach my $xkey (@xkeys) {
		foreach my $hkey (@hkeys) {
			if ( $hkey eq $xkey ) {
				next XKEY;
			}
		}

		# Whoa!! No match found in the (dot)H file!
		if ( length( $p{msg} ) > 0 ) {

			# start w/tab if after 1st err msg
			$p{msg} .= "\t";
		}

		# Accum err diag msgs
		$p{msg} .= "(dot)H file missing item \'$xkey\'\n";
		$errflg = -1;
	}

	if ( $errflg == -1 ) {
		printmsg();
		return (-1);
	}

	return (0);
}
###------  end  chkXMLmatchInDotH  -----------------------

###------  start  readin_Xfile  -----------------------
sub readin_Xfile {

	our $dotX_fn;
	our $Description;
	our $InfoURL;
	our $PolicyURL;
	our $Ack;
	our (%H,%DF,%Name,%Units,%VMin,%VMax,%Fill,%Dim,%SCtype,%Legends,%Description);
	my $ii = 0;
	my $jj = 0;

	# Does XML file exist
	if ( filexists($dotX_fn) ) {
		$p{msg} = "Does NOT Exist: $dotX_fn\n\n";
		printmsg();
		return (-1);
	}
	if ( !-r $dotX_fn ) {    # Is it readable...
		$p{msg} = "Is NOT Readable: $dotX_fn\n\n";
		printmsg();
		return (-1);
	}

	# Read XML file
	my $xml = new XML::Simple;

	#    my $cmd  = $xml->XMLin("NumericalData-ACE-SWEPAM-HDF_PT64S.xml");
	my $cmd  = $xml->XMLin($dotX_fn);
	my $data = $cmd;

	# extract global metadata
	$Description = $data->{NumericalData}->{ResourceHeader}->{Description};
	$InfoURL = $data->{NumericalData}->{ResourceHeader}->{InformationURL}->[0]->{URL};
	$PolicyURL = $data->{NumericalData}->{ResourceHeader}->{InformationURL}->[1]->{URL};
	$Ack = $data->{NumericalData}->{ResourceHeader}->{Acknowledgement};

	# How many PhysicalParameters?
	my $npparams = scalar( @{ $data->{NumericalData}->{PhysicalParameter} } );

	# Extract param_keys and display_formats
	for ( $ii = 0 ; $ii < $npparams ; $ii++ ) {
		my $pkey =
		  $data->{NumericalData}->{PhysicalParameter}->[$ii]->{ParameterKey};

		my $dformat =
		  $data->{NumericalData}->{PhysicalParameter}->[$ii]->{RenderingHints}->{ValueFormat};
		$DF{$pkey} = $dformat;

		my $sctype =
		  $data->{NumericalData}->{PhysicalParameter}->[$ii]->{RenderingHints}->{ScaleType};
		$SCtype{$pkey} = $sctype;

		my $nm =
		  $data->{NumericalData}->{PhysicalParameter}->[$ii]->{Name};
		$Name{$pkey} = $nm;

		my $un =
		  $data->{NumericalData}->{PhysicalParameter}->[$ii]->{Units};
		$Units{$pkey} = $un;

		my $vmi =
		  $data->{NumericalData}->{PhysicalParameter}->[$ii]->{ValidMin};
		$VMin{$pkey} = $vmi;

		my $vma =
		  $data->{NumericalData}->{PhysicalParameter}->[$ii]->{ValidMax};
		$VMax{$pkey} = $vma;

		my $fi =
		  $data->{NumericalData}->{PhysicalParameter}->[$ii]->{FillValue};
		$Fill{$pkey} = $fi;
		
		my $de =
		  $data->{NumericalData}->{PhysicalParameter}->[$ii]->{Description};
		$Description{$pkey} = $de;
		
		my $dim =
		  $data->{NumericalData}->{PhysicalParameter}->[$ii]->{Structure}->{Size};
		if (defined $dim) {
		   $Dim{$pkey} = $dim;
		   my @flds;
		   for ($jj=0;$jj<$dim;$jj++) {
#		      print STDERR "$data->{NumericalData}->{PhysicalParameter}->[$ii]->{Structure}->{Element}->[$jj]->{Name}   ";
		      @flds = split " ", $data->{NumericalData}->{PhysicalParameter}->[$ii]->{Structure}->{Element}->[$jj]->{Name};
#		      $Legends{$pkey}[$jj] = $flds[2];
		      $Legends{$pkey}[$jj] = "$flds[1]_$flds[2]";
		   }
#		   for ($jj=0;$jj<$dim;$jj++) {
#		      print STDERR "$Legends{$pkey}[$jj]   ";
#		   }
#		   print STDERR "\n";
		} else {
		   $Dim{$pkey} = 1;
		   $Legends{$pkey} = $data->{NumericalData}->{PhysicalParameter}->[$ii]->{Name};
		}
		
	}
	return (0);
}
###------  end  readin_Xfile  -----------------------

###------  start  readin_Hfile  -----------------------
sub readin_Hfile {
	our @HdataType = ();
	our @HparmName = ();
	our @Hindex    = ();
	our $Hcount    = 0;
	our $dotH_fn;
	our %H;

	my @flds      = ();
	my $linein    = "";
	my $lineIndex = -1;

	# Does file exist
	if ( filexists($dotH_fn) ) {
		$p{msg} = "Does NOT Exist: $dotH_fn\n\n";
		printmsg();
		return (-1);
	}
	if ( !-r $dotH_fn ) {    # Is it readable...
		$p{msg} = "Is NOT Readable: $dotH_fn\n\n";
		printmsg();
		return (-1);
	}
	if ( !open FN, "<$dotH_fn" ) {
		$p{msg} = "Can't open $dotH_fn\n\t$!\n\n";
		printmsg();
		return (-1);
	}
	while ( $linein = <FN> ) {
		chomp $linein;
		$lineIndex++;
		@flds = split " ", $linein, 3;
		next if ( $#flds < 0 );

		# Is Physical Parameter??
		if (   ( $flds[0] =~ /^int\d+/ )
			or ( $flds[0] =~ /^float\d+/ )
			or ( $flds[0] =~ /^uint\d+/ ) )
		{
			$flds[1] =~ s/\;$//;
			$flds[1] =~ s/\[.*\]$//;
# print "$flds[1]\n";			

			#=======
			# ONLY testing ONLY...ignores any phy.params w/"proton" in name
			#	if($dbug) {
			#	  next if($flds[1]=~proton); # ignore this line
			#	}
			#=======

			#  DATA TYPE   NAME    INDEX into File
			#   $flds[0], $flds[1], $lineIndex );
			push @HdataType, $flds[0];
			push @HparmName, $flds[1];
			push @Hindex,    $Hcount;
			$H{ $flds[1] } = $Hcount;
			$Hcount++;
			next;
		}
	}
	close FN;
	$Hcount = $#HdataType;

	return (-1) if ( $Hcount < 0 );    # No lines!!
	return (0);                        # OK
}
###------  end  readin_Hfile  -----------------------

###------  start  filexists  -----------------------
sub filexists {

	#
	# SYNTAX:      $rtn = filexists(EXPR);
	# Returns 0 if file EXPR exists, else returns -1.
	#
	# No whinning work-around for `ls foo*` when no file found
	#    and PERL -e file test returning false when there is a file(linked??)
	return (-1) if ( !defined( $_[0] ) );

	my @f = ();
	chomp $_[0];    # just to be sure
	@f = <$_[0]>;   # Using PERL File Globbing Operator
	if ( $#f >= 0 ) {
		return (0);    # OK, at least one file exists
	}
	else {
		return (-1);    # Oops, No files found
	}
}
###------  end  filexists  -----------------------

###------  start  print2op  -----------------------
sub print2op {
    # Print the name of each XML data, but in same order as H file
	our $Description;
	our $InfoURL;
	our $PolicyURL;
	our $Ack;
	our (%H,%DF,%Name,%Units,%VMin,%VMax,%Fill,%Dim,%SCtype,%Legends,%Description);


	my @h_keys  = keys(%H);    # %H param names
	my $h_count = $#h_keys;

	my @xpkeys       = keys(%DF);       # %DF param names
	my $x_count      = $#xpkeys;
	my $x_countPlus1 = $x_count + 1;
	
	my $datnam = "";
    my $xpkey = "";
    my @op = ();

    # PRINT:  LINE COUNT (base one)	
	printf STDOUT ( "%d\n", $x_countPlus1 );

    # Print global metadata
	printf STDOUT ( "4\n");
	printf STDOUT ( "%s\n", $Description );
	printf STDOUT ( "%s\n", $Ack );
	printf STDOUT ( "%s\n", $InfoURL );
	printf STDOUT ( "%s\n", $PolicyURL );
    
	# First build a string array @op: "<indx_nmbr datnam fmt>"
    #       (...so can later sort it...)

	# Find %H index val for each data name in %DF 
	for ( my $xi = 0 ; $xi <= $x_count ; $xi++ ) {    # Loop thru %DF
		$datnam = $xpkeys[$xi];   # param name from %DF

        # ... building the string array @op: "<indx_nmbr datnam fmt>"
        # Get directly the index value in %H via SAME datnam found in the %DF
 	    my $line = sprintf "%3d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%3d", $H{$datnam}, $datnam, $DF{$datnam},
		$Name{$datnam}, $Units{$datnam}, $VMin{$datnam}, $VMax{$datnam}, $Fill{$datnam}, $SCtype{$datnam},
		 $Description{$datnam},$Dim{$datnam};
            if ($Dim{$datnam} >1) {
	       for (my $iii = 0; $iii<$Dim{$datnam}; $iii++) {
	          $line = sprintf "%s\t%s",$line,$Legends{$datnam}[$iii];
	       }
            } else {
	       $line = sprintf "%s\t%s",$line,$Legends{$datnam};
	    }
#	    print "$datnam\n";
	    push @op, $line;
	}

	# Sort the @op array & Print it out
	my @sorted = sort @op;
	my $i = 0;
	
	while ( $i <= $#sorted ) {
		my ($fmt, $name, $units, $vmin, $vmax, $fill, $sctype, $dim, $desc,$legends) = ("","","","","","","","","","");
		my $idx = 0;
		( $idx, $datnam, $fmt, $name, $units, $vmin, $vmax, $fill, $sctype, $desc, $dim, $legends ) = split "\t", $sorted[$i], 12;
		my $tab1 = "";    # make look nice
		if ( length $datnam < 8 ) {   # Chk for short name
			$tab1 = "\t";
		}
		printf STDOUT ( "%s\t%s%s\t%d\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
			       $datnam, $tab1, $fmt, $idx, $name, $units, $vmin, $vmax, $fill, $sctype, $desc, $dim, $legends);
		$i++;
	}
	return (0);
}
###------  end  print2op  -----------------------

###------  start  chkCmdline  -----------------------
sub chkCmdline {

	if ( $#ARGV != $nargs ) {    # of args(less one) expect.on cmdline
		my $argv_plus  = $#ARGV + 1;
		my $nargs_plus = $nargs + 1;
		$p{msg} =
		  sprintf( "Cmdline args:\n\tFound: \t\t%d\n\tExpected: \t%d.\ndotH XML\n\n",
			$argv_plus, $nargs_plus );
		printmsg();
		return (-1);
	}

	return (0);
}
###------  end  chkCmdline  -----------------------

###------  start  printmsg  -----------------------
sub printmsg {
	my $msglevel = "ERR-";
	$msglevel = "Warning-" if ( uc( $_[0] ) eq "W" );
	printf STDERR Carp::longmess("\n$msglevel $0:\n\t$p{msg}\n");
	return;
}
###------  end  printmsg  -----------------------

###------  start  newsubname  -----------------------
sub newsubname {

	return;
}
###------  end  newsubname  -----------------------