#!/usr/bin/perl

use strict;
use lib '/cdf/scratch/cdfopr/danielw/CommonLibs';

require "db_queries.pl";
require "CalibValidFiles.pl";
require "CalibStatus.pl";
#require "CommonDefs.pl";

my $status_dir = "/cdf/www/offline/calibrations/Status";
my $status_html = "http://www-cdf.fnal.gov/offline/calibrations/Status";

my $warning_email = "boj\@fnal.gov";

my %streams;
$streams{"pes-align"} = "b";
$streams{"pes-gain"} = "g";
$streams{"emtiming"} = "b";
$streams{"hadtiming"} = "g";
$streams{"cal-had"} = "j";
$streams{"miniplug"} = "g";
$streams{"cal-em"} = "b";
$streams{"tof"} = "g";
$streams{"badchannels"} = "g";

#$ENV{'ORACLE_HOME'} = '/home/cdfsoft/products/oracle_client/v9_2_0_1a/Linux+2';
$ENV{'ORACLE_HOME'} = $ENV{'CDFSOFT2_DIR'}.'/../../../products/oracle_client/v9_2_0_1a/Linux+2';
#$ENV{'TNS_ADMIN'} = '/home/cdfsoft/dist/oracle_tnsnames';
$ENV{'TNS_ADMIN'} = $ENV{'CDFSOFT2_DIR'}.'/../../oracle_tnsnames';

use DBI;

my $dbh = DBI->connect('dbi:Oracle:cdfonprd','cdf_reader','reader') 
	or die "Error connecting $DBI::errstr\n";

my $svx_txt = '';
my $cot_txt = '';
my $beamlines_txt = '';
my $calib_txt = '';

my $donecolor="green";
my $missingcolor="red";
my $notyetcolor="blue";

my $firstrun = 217989;
my $lastrun  = 999999;

my @rungroups = &getReadyRungroups();

my $dfree = &getSpaceLeft();

my %current_jobs = &getAllSamgliaJID();

#printf("Looking for files");
printf("<!-- Querying file listsings for output -->\n");
my %cfiles_proc_run = &getNumCalibValidFiles();

# get list of ready rungroups [ not yet submitted ]
#printf("Getting rungroups...\n\n\n\n");

my @runsv;
printf("<!-- Querying db for goodruns -->\n");
&get_good_runs($dbh,$firstrun,$lastrun,\@runsv);
my @runs = reverse @runsv;

my $maxrun=$runs[0];
my $minrun=$runs[ (scalar @runs)-1 ];

#printf("<!-- good runs: @runs -->\n");

printf("<!-- Querying SAM from runs $minrun $maxrun -->\n");
my %cfilesmax_stream_run = &getAllMaxCalibFiles($minrun,$maxrun);

#    printf("<!-- All valid keys are: @keysc -->\n");

my $run;
my @rstatus;
my @link;
my $datestring = `date`;
chop($datestring);
  
printf("<center>Updated: <font color=green>$datestring</font> | Earliest run considered: <font color=green>$firstrun</font><br>Disk space on fcdfdata125:/export/data1,2: <font color=green>$dfree</font> <br>Color Key: <font color=$missingcolor>missing</font> | <font color=$donecolor>done</font><br><br><br>\n");

my $proc="view_summary";
if ($dfree =~ /100/ || $dfree =~ /99/) 
{
    my $message = "Warning: disk space is low on fcdfdata125!\n. See: http://www-cdf.fnal.gov/offline/calibrations/Summary.html";
    my $recip = $warning_email;
    open(MOUT,">.mail.${proc}.$$");
    print MOUT $message;
    close(MOUT);
    my $res = `mail -s \"Calibrations: WARNING disk space low\" $recip < .mail.${proc}.$$`;
    `rm .mail.${proc}.$$`;
#    printf("Sent \"$message\" to $recip with result $res\n");
}

printf("<table cellSpacing=0 cellPadding=4 border=1>");
printf("<tr><th> Run Range</th><th>pass 10A<br>built</th><th>SVX,COT<br>Beamlines</th><th>pass 17A<br>built</th><th>Subdetector Calibration</th><th>pass 17test1<br>built</th><th>Validation</th><th>Plots</th><th>pass 17<br>built</th>\n");


my %streamBdone;
my %streamGdone;
my %streamJdone;
my $latencyB = 9999;
my $latencyG = 9999;
my $latencyJ = 9999;
my $runningB=0;
my $runningG=0;
my $runningJ=0;
my %pass10Abuilt;
my %pass17Abuilt;
my %pass17built;
my %svxbeamsbuilt;
my %cotbeamsbuilt;
my %pass17test1built;
my %totalruns;

my %proccfiles;
my %maxcfiles;
my %procpending;
my %usedsetvalid;

my %stripvfiles;

my @processes = ("pes-align","emtiming","cal-em","cal-had","pes-gain","hadtiming","miniplug","tof","badchannels");
my @strips = ("zee","wen","zmm","wmn","jpsi","mb");
my @plots = ("zee","zmm","cal-em","mb","dqm","emtiming","pes-gain","pes-align","plug-energy-scale","miniplug","tof","jpsi");

my %usedsets;
$usedsets{"pes-align"} = "L3_PHYSICS_SMX";

foreach $run (@runs)
{
    


    $datestring= `date`;
    chop($datestring);
    printf("<!-- [ $datestring ] Starting sort of run $run --!>\n");
    my $i=0;
    $link[$i] = "";

    my $rungroup = &getRunGroup($run);
    if ($rungroup eq 0) { $rungroup = "current"; }

    printf("<!-- [ $datestring ]  Run $run in group $rungroup --!>\n");

    $totalruns{$rungroup}++;
    my $age = &get_run_age( $dbh, $run);

    if (&check_pass( $dbh, $run, "10A") ) { $pass10Abuilt{$rungroup}++; }
    if (&check_pass( $dbh, $run, "17TEST1") ) { $pass17test1built{$rungroup}++; }
    if (&check_pass( $dbh, $run, "17") ) { $pass17built{$rungroup}++; }
    if (&check_pass( $dbh, $run, "17A") ) { $pass17Abuilt{$rungroup}++; }
    
    if (&check_process( $dbh, $run, "PAD_PHYSICS_COTBEAM") ) { $svxbeamsbuilt{$rungroup}++; }
    if (&check_process( $dbh, $run, "PAD_PHYSICS_SVXBEAM") ) { $cotbeamsbuilt{$rungroup}++; }
    

    my $proc;

    printf("<!-- [ $datestring ] checking checking files --!>\n");
    my %any_files;

    foreach $proc (@processes)
    {

	my $tag = $proc.".".$run;
	my $rgtag = $rungroup.".".$proc;

#	printf("<!-- [ $datestring ] checking checking files $proc tag is $tag  in stream $streams{$proc}--!>\n");

	$maxcfiles{$rgtag} += $cfilesmax_stream_run{$streams{$proc}.".".$run};

	# +1 -1 to convert "" to 0
	$proccfiles{$rgtag} += $cfiles_proc_run{$tag} +1 -1;
	
	if ($proccfiles{$rgtag} > 0) { $any_files{$streams{$proc}} = 1; }

	if (&check_process( $dbh, $run, $usedsets{$proc} ) ) 
	{ 	$usedsetvalid{$rgtag}++; }
	else
	{ 	$usedsetvalid{$rgtag} = $usedsetvalid{$rgtag} +1-1; }

    }

    if ($current_jobs{"bphysr.".$run} ne "") { $runningB++;}
    if ($current_jobs{"gphysr.".$run} ne "") { $runningG++;}
    if ($current_jobs{"jphysr.".$run} ne "") { $runningJ++;}

    printf("<!-- [ run $run files? ".$any_files{"b"}." age $age latency $latencyB -->");
    if ($any_files{"b"}) { $streamBdone{$rungroup}++; if ($age < $latencyB) {$latencyB=$age;} }
    if ($any_files{"g"}) { $streamGdone{$rungroup}++; if ($age < $latencyG) {$latencyG=$age;} }
    if ($any_files{"j"}) { $streamJdone{$rungroup}++; if ($age < $latencyJ) {$latencyJ=$age;} }
    
    my $strip;
    $i=0;
    foreach $strip (@strips)
    {
#	printf("<!-- [ $datestring ] checking checking files $strip --!>\n");

	my $tag = $strip.".".$run;
	my $rgtag = $rungroup.".".$strip;

	# +1 -1 to convert "" to 0
	$stripvfiles{$rgtag} += $cfiles_proc_run{$tag} +1 -1;

    }

}


    ##########################
    ## produce the table

my $rungroup;
my @rkeys = keys %pass10Abuilt;
reverse @rkeys;
foreach  $rungroup (@rkeys) 
{
    
   
	printf("<!-- [ $datestring ] making table entry for $rungroup --!>\n");

	printf("<tr>");
	printf("<td>$rungroup</td>\n");
	
	my $color = $donecolor;
	if ($pass10Abuilt{$rungroup}<$totalruns{$rungroup})
	{
	    { $color = $missingcolor; }
	}
	printf("<td><font color=$color>$pass10Abuilt{$rungroup}</font>/$totalruns{$rungroup}</td>\n");
	
	my $color = $donecolor;
	if ($svxbeamsbuilt{$rungroup}<$totalruns{$rungroup})
	{
	    { $color = $missingcolor; }
	}
	printf("<td><font color=$color>$svxbeamsbuilt{$rungroup}</font>/$totalruns{$rungroup}<br>");

	my $color = $donecolor;
	if ($cotbeamsbuilt{$rungroup}<$totalruns{$rungroup})
	{
	    { $color = $missingcolor; }
	}
	printf("<font color=$color>$cotbeamsbuilt{$rungroup}</font>/$totalruns{$rungroup}</td>\n");

	my $color = $donecolor;
	if ($pass17Abuilt{$rungroup}<$totalruns{$rungroup})
	{
	    { $color = $missingcolor; }
	}
	printf("<td><font color=$color>$pass17Abuilt{$rungroup}</font>/$totalruns{$rungroup}</td>\n");
	
	
	printf("<td>");
	printf("<table border=0>");
	printf("<tr><th>Subdet.</th><th>Files</th><th>Results</th><th>dB</th></tr>\n\n");
	my $proc;
	foreach $proc (@processes)
	{
	    my $tag = $rungroup.".".$proc;
	    
	    $color = $donecolor;
	    if ($proccfiles{$tag}<$maxcfiles{$tag})
	    {
		$color = $missingcolor;
	    }
	    
	    printf("<tr><td><b>${proc}</b></td><td><font color=$color>$proccfiles{$tag}</font>/$maxcfiles{$tag}</td>\n");
	    printf("<td>");

	    if ($rungroup ne "current")
	    {
		my $procres = &getStatusRungroupProc("Calib","Pending",$rungroup,$proc);
		
		if ($procres =~ /Error/)
		{
		    printf("<font color=$missingcolor>ERRORS</font>"); 

		}
		else
		    {
			if ($procres =~ /Ready/)
			{
			    printf("<a href=\"$status_html/Calib_Pending/$rungroup/$proc/index.html\"><font color=$donecolor>Ready</font></a>");
			}
			else 
			{
			    printf("<!--- Status for $rungroup $proc is >$procres< --!>\n");
			}
		    }
	    }
	    	    
	    printf("</td>");

	    # used sets must have finished calibrations done first
	    $color = $donecolor;
	    if ($usedsetvalid{$tag}<$totalruns{$rungroup})
	    {
		$color = $missingcolor; 
	    }
	    printf("<td><font color=$color>$usedsetvalid{$tag}</font>/$totalruns{$rungroup}</td></tr>\n");
	
	    
	   
	    
	}
	printf("</table>");

	if ($rungroup eq "current")
	{

	    if ($latencyB > 100 || $latencyG > 100 || $latencyJ > 100)
	    {
		my $message = "Warning: CalibExe has fallen >= 100 hrs behind current data taking.\nSee: http://www-cdf.fnal.gov/offline/calibrations/Summary.html\n\nStream\tDone\tLatency\tRunning\nb\t\t$streamBdone{$rungroup}/$totalruns{$rungroup}\t$latencyB hrs\t$runningB jobs\ng\t\t$streamGdone{$rungroup}/$totalruns{$rungroup}\t$latencyG hrs\t$runningG jobs\nj\t\t$streamJdone{$rungroup}/$totalruns{$rungroup}\t$latencyJ hrs\t$runningJ jobs\n";
		
		my $recip = $warning_email;
		open(MOUT,">.mail.${proc}.$$");
		print MOUT $message;
		close(MOUT);
		my $res = `mail -s \"CalibExe Latencies are high\" $recip < .mail.${proc}.$$`;
		`rm .mail.${proc}.$$`;
#		printf("Sent \"$message\" to $recip with result $res\n");
	    }


	    printf("<br><hr><br><table border=0><tr><th>Stream</th><th>Done</th><th>Latency</th><th>Running</th></tr>");
	    printf("<tr><td><b>b</b></td><td>$streamBdone{$rungroup}/$totalruns{$rungroup}</td><td>$latencyB hrs</td><td>$runningB jobs</td></tr>");
	    printf("<tr><td><b>g</b></td><td>$streamGdone{$rungroup}/$totalruns{$rungroup}</td><td>$latencyG hrs</td><td>$runningG jobs</td></tr>");
	    printf("<tr><td><b>j</b></td><td>$streamJdone{$rungroup}/$totalruns{$rungroup}</td><td>$latencyJ hrs</td><td>$runningJ jobs</td></tr>");
	    printf("</table><br>");
	}
	printf("</td>\n\n");

	my $color = $donecolor;
	if ($pass17test1built{$rungroup}<$totalruns{$rungroup})
	{
	    { $color = $missingcolor; }
	}
	printf("<td><font color=$color>$pass17test1built{$rungroup}</font>/$totalruns{$rungroup}</td>\n");

	
	printf("<td>");
	printf("<table border=0>");
	printf("<tr><th>Stream</th><th>Files</th><th>Status</th></tr>\n");

	my $strip;
	foreach $strip (@strips)
	{
#	printf("<!-- strip $strip i=$i files = $stripvfiles[$i] pending= $strippending[$i] res = $stripres[$i]-->\n");
	    
	    my $tag = $rungroup.".".$strip;
	    
	    $color = $donecolor;
	    my $badcolor=$notyetcolor;
	    
	    # strip name and number of stripped files
	    if ($stripvfiles{$tag} ne 0)
	    {
		printf("<tr><td><b>${strip}</b></td><td><font color=$donecolor>$stripvfiles{$tag}</font></td>\n");
	    }
	    else
	    {
		printf("<tr><td><b>${strip}</b></td><td><font color=$badcolor>$stripvfiles{$tag}</font></td>\n");
	    }
	    

	    if (!$rungroup ne "current")
	    {
		my $stripres = &getStatusRungroupProc("Valid","Submitted",$rungroup,$strip);
		
		if ($stripres ne "")
		{
		    printf("<td><a href=\"$status_html/Valid_Submitted/$rungroup/$strip/index.html\"><font color=$donecolor>$stripres</font></a></td>\n");
		}
		else
		{
		    printf("<!--- Status for $rungroup $strip is >$stripres< --!>\n");

		    printf("<td>None</td>");
		}

	    }
	}


	printf("</table><td>");
	printf("<table border=0>");
	printf("<tr><th>Plots</th><th>Ntuples</th><th>Status</th></tr>\n");
	my $plot;
	foreach $plot (@plots)
	{

	    my @vfiles_run_strip = &getValidNtuples($plot,$rungroup);

	    my $n_ntuples = 1-1+scalar @vfiles_run_strip;
	    printf("<tr><td><b>$plot</b></td>");
	    if ($n_ntuples)
	    {
		printf("<td><font color=$donecolor>$n_ntuples</font></td>\n");
	    }
	    else
	    {
		printf("<td><font color=$missingcolor>$n_ntuples</font></td>\n");
	    }

	    my $ntupleres = &getStatusRungroupProc("Valid","Plots",$rungroup,$plot);
		
	    if ($ntupleres ne "")
	    {
		printf("<td><a href=\"$status_html/Valid_Plots/$rungroup/$plot/index.html\"><font color=$donecolor>$ntupleres</font></a></td>\n");
	    }
	    else
	    {
		printf("<td><font color=$missingcolor>None</font></td>");
	    }
	    
	    
	}
	printf("</td></tr>\n");

	printf("</table></td>\n\n");


	my $color = $donecolor;
	if ($pass17built{$rungroup}<$totalruns{$rungroup})
	{
	    { $color = $missingcolor; }
	}
	printf("<td><font color=$color>$pass17built{$rungroup}</font>/$totalruns{$rungroup}</td>");

    printf("</tr>\n\n");

}
    printf("</table></center>");

$dbh->disconnect;

exit;

sub check_pass
{
    #i) db handle ii) run iii) pass name vi) jobset name v) time offset in hours 
    
    # this checks that a run is not in a pass AND (if specified) is missing a jobset
    # returns a string if missing
    
    my ( $dbh, $run, $pass ) = @_;

    my $pass_index = &get_pass_index( $dbh, $pass );

    my $sth = $dbh->prepare( "select lorun from passcalibs 
                          where lorun = ? and pass_index = ?" );

    $sth->execute( $run, $pass_index );
    if ( $sth->fetchrow ) { return 1 };
    return 0;
}

sub check_process
{
    
    my ( $dbh, $run, $process_name ) = @_;
    
    
    my $latest = &get_latest_version( $dbh, $run, $process_name );

    return $latest;
}

sub get_good_runs
{
# returns an array of good runs (farms definiton minus csl_summary) 

# Arguments: 1) database handle 2) start run 3 array to return runs 
# fills hash with key = run and value = run terminate time 

  my ( $dbh, $start_run, $end_run,$run) = @_;

  my $sql = 
  "select rc.runnumber
  from runconfigurations rc, run_status rs, physicstables pt
  where rs.runcontrol_status != 0 
    and rs.shiftcrew_status != 0
    and rc.b0integratedlivelumi > 10.0
    and rc.csl_neventsoutput > 1000
    and not pt.physicstablename like '%TEST%'
    and (pt.physicstablename like '%Phyiscs' or pt.physicstablename like '%PHYSICS%')
    and pt.runnumber = rc.runnumber
    and rs.runnumber = rc.runnumber
    and rc.runnumber >= ?
    and rc.runnumber <= ?
  order by rc.runnumber"; 

  my $sth = $dbh->prepare($sql);
  $sth->execute($start_run,$end_run);
  
#  printf("request = $sql\n");
  
  while (my $row = $sth->fetchrow) { push @$run, $row }

  return 1;
}
