
use strict;
my %fileprefix;
my %fileprefix_out;
my %filepostfix;
my %fileclass;
my %filestream;
my %params;

##my $firstrun = 228590;
#my $firstrun = 228596;
my $firstrun = 254689;


#my $basedir="$ENV{'HOME'}/data2/concatenated/current/";
#my $scriptdir="$ENV{'HOME'}/data2/concatenated/scripts/";
#my $srcdir="$ENV{'HOME'}/data1/current/";
my $basedir="$ENV{'base_dir'}";
my $scriptdir="$ENV{'script_dir'}";
my $srcdir="$ENV{'src_dir'}";

# tested and working

$fileprefix{"cal-em"} = "cemcal\_bb";
$filepostfix{"cal-em"} = "calb\.text";
$fileclass{"cal-em"} = "text";
$filestream{"cal-em"} = "b";

$fileprefix{"cal-mb"} = "mbscal\_gb";
$filepostfix{"cal-mb"} = "calb\.text";
$fileclass{"cal-mb"} = "text";
$filestream{"cal-mb"} = "g";

$fileprefix{"pes-align"} = "pesal\_bb";
$filepostfix{"pes-align"} = "calb\.root";
$fileclass{"pes-align"} = "tree";
$params{"pes-align"} = "PesAlignment";
$filestream{"pes-align"} = "b";

$fileprefix{"emtiming"} = "emtimingntuple\_bb";
$filepostfix{"emtiming"} = "calb\.root";
$fileclass{"emtiming"} = "tree";
$params{"emtiming"} = "CalorTimeNtuple";
$filestream{"emtiming"} = "b";

$fileprefix{"hadtiming"} = "hadtimingntuple\_gb";
$filepostfix{"hadtiming"} = "calb\.root";
$fileclass{"hadtiming"} = "tree";
$params{"hadtiming"} = "CalorTimeNtuple";
$filestream{"hadtiming"} = "g";

$fileprefix{"calibhist"} = "calibhist_gb";
$filepostfix{"calibhist"} = "calb\.root";
$fileclass{"calibhist"} = "histo";
$filestream{"calibhist"} = "g";


$fileprefix{"tof"} = "calibhist_gb";
$fileprefix_out{"tof"} = "tofcalib_gb";
$filepostfix{"tof"} = "calb\.root";
$fileclass{"tof"} = "2trees";
$params{"tof"} = "TofCalibMenuModule-LightSpeed/tof_calib TofCalibMenuModule-TimeWalk/tof_calib";
$filestream{"tof"} = "g";

#$fileprefix{"miniplug"} = "mpstntuple\_gb";
#$filepostfix{"miniplug"} = "calb\.root";
#$fileclass{"miniplug"} = "stn";
#$params{"miniplug"} = "";
#$filestream{"miniplug"} = "g";

$fileprefix{"cpr2"} = "cpr2stntuple\_gb";
$filepostfix{"cpr2"} = "calb\.root";
$fileclass{"cpr2"} = "stn";
$params{"cpr2"} = "";
$filestream{"cpr2"} = "g";

#connected to disk writing
$fileprefix{"cal-had-raw"} = "jpsimumu_jb";
$filepostfix{"cal-had-raw"} = "calb\.root";
#$params{"cal-had-raw"} = "/export/data2/cdfopr/concatenated/current/streamj/tosam.list caljp0";
$params{"cal-had-raw"} = "${basedir}/streamj/tosam.list caljp0";
$fileclass{"cal-had-raw"} = "sam";
$filestream{"cal-had-raw"} = "j";

$fileprefix{"zee"} = "zeestrip_bb";
$filepostfix{"zee"} = "calb\.root";
$fileclass{"zee"} = "cdf";
$filestream{"zee"} = "b";

#$fileprefix{"mb"} = "mbstrip_gb";
#$filepostfix{"mb"} = "calb\.root";
#$fileclass{"mb"} = "cdf";
#$filestream{"mb"} = "g";

$fileprefix{"mb"} = "mbstrip_gb";
$filepostfix{"mb"} = "calb\.root";
$fileclass{"mb"} = "sam";
#$params{"mb"} = "/export/data2/cdfopr/concatenated/current/streamg/tosam.list calgp0";
$params{"mb"} = "${basedir}/streamg/tosam.list calgp0";
$filestream{"mb"} = "g";

$fileprefix{"zmm"} = "zmmstrip_bb";
$filepostfix{"zmm"} = "calb\.root";
$fileclass{"zmm"} = "cdf";
$filestream{"zmm"} = "b";

$fileprefix{"wen"} = "wenstrip_bb";
$filepostfix{"wen"} = "calb\.root";
$fileclass{"wen"} = "cdf";
$filestream{"wen"} = "b";

$fileprefix{"wmn"} = "wmnstrip_bb";
$filepostfix{"wmn"} = "calb\.root";
$fileclass{"wmn"} = "cdf";
$filestream{"wmn"} = "b";

$fileprefix{"cal-had"} = "hadlerstntuple_jb";
$filepostfix{"cal-had"} = "calb\.root";
$fileclass{"cal-had"} = "stn";
$filestream{"cal-had"} = "j";

$fileprefix{"pesgain"} = "pesgain\_gb";
$filepostfix{"pesgain"} = "calb\.root";
$fileclass{"pesgain"} = "2trees";
$params{"pesgain"} = "PesGainsNtuple PesOccupancyNtuple";
$filestream{"pesgain"} = "g";


####
# warning: pesocc must come after pesgain, since they use the same concated files and pesgain is kludged not to delete old files for this reason

# all 
#remove miniplug and add cpr2
#my @types = ("calibhist","tof","cal-had","zee","zmm","wmn","wen","cal-em","emtiming","pes-align","cal-mb","hadtiming","miniplug","cal-had-raw","pesgain","mb");
my @types = ("calibhist","tof","cal-had","zee","zmm","wmn","wen","cal-em","emtiming","pes-align","cal-mb","hadtiming","cpr2","cal-had-raw","pesgain","mb");

# testing modes
#my @types=("cal-had");
#my @types=("cal-had-raw");
#my @types=("cal-em","cal-had-raw","cal-had","zee","pes-align","miniplug","calibhist");
#my @types = ("mb");

my $type;

my $lockfile = "${basedir}/concat.lockfile";

sub handleLock() {
    my $llock = `ls $lockfile`;
    chop($llock);
    if ( $llock ne "" ) {
	printf("[WARNING] Found existing lock file [ $lockfile ] Exiting.\n");
	exit(0);
    }
    else {
	printf("[INFO] Creating a new lock.\n");
	`touch $lockfile`;
    }
}

sub removeLock() 
{
	 printf("[INFO] Removing lock.\n");
    `rm $lockfile`;
}


sub logFileName()
{
    my ($run,$type) = @_;

    return ${basedir}."stream".$filestream{$type}."/".$type.".".$run.".log";
}


sub listFileName()
{
    my ($run,$type) = @_;

    return ${basedir}."/stream".$filestream{$type}."/".$type.".".$run.".list";
}

sub concatFileName()
{
    my ($run,$type,$ndone,$checkExists) = @_;

    my $outName = "";
    if ($fileprefix_out{$type} ne "")
      {
	$outName = ${basedir}."stream".$filestream{$type}."/".$fileprefix_out{$type}.".".$run.".n".$ndone.".".$filepostfix{$type}; }
    else
      {
	$outName = ${basedir}."stream".$filestream{$type}."/".$fileprefix{$type}.".".$run.".n".$ndone.".".$filepostfix{$type};
      }


    #check whether a given file exists (if requested), if not return empty string
    if ($checkExists == 0)  {
      return $outName;
    }
    else  {
      if ( -e $outName )  {
	return $outName;
      }
      else  {
	return ""
      }
    }
}

sub concatFileNameWildcard()
{
    my ($run,$type) = @_;

    return ${basedir}."stream".$filestream{$type}."/".$fileprefix{$type}.".".$run.".*.".$filepostfix{$type};
}


my $datestring = `date`;
chop($datestring);

printf("CalibExe Concatenator. $datestring\n");

&handleLock();

for $type (@types)
{
    my $datestring = `date`;
    chop($datestring);
    printf("[ $datestring ] Looking for new $type files\n");
    my @files = `ls ${srcdir}/stream$filestream{$type} | grep $fileprefix{"$type"} | grep -v "moved_to_sam"`;
    my $file;
    my %files_todo_run;
    my %files_ntodo_run;
    my %files_tomark_run;
    my %files_done_run;
    foreach $file (@files)
    {
	chop($file);
	my $run; 
	my $sec;
#	printf("File $srcdir/$file prefix $fileprefix{$type} postfix $filepostfix{$type}\n");
	($run,$sec) = ($file =~ /$fileprefix{"$type"}([0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f])\.([0-9a-f][0-9a-f][0-9a-f][0-9a-f])$filepostfix{"$type"}/);
	my $drun = hex $run;
#	printf("File $srcdir/$file prefix $fileprefix{$type} run $run ($drun) section $sec postfix $filepostfix{$type}\n");

	if ($run eq "" || $sec eq "")
	{
	    printf("Couldn't determine run/sec ($run/$sec) for file $file\n");
	}
	else
	{
	
	    # make sure run is in period
	    if ($drun > $firstrun)
	    {
		
#		printf("----Using file in run $drun > $firstrun\n");
		# see if we've done this one already
		my $lfile = &listFileName($run,$type);
		my @donefiles = `cat $lfile | grep $fileprefix{$type} 2> /dev/null`;
		
		my $done;
		my $doneyet = 0;
		$files_done_run{$run} = (scalar @donefiles);
		foreach $done (@donefiles)
		{
		    chop($done);
		    if ($done =~ /$file/)
		    {
			#printf("File $file already concatenated. Skipping\n");
			#`rm ${srcdir}/stream$filestream{$type}/$file`;
			$doneyet =1;
		    }
		}
		
		if ($doneyet==0)
		{
		    $files_todo_run{$run} = $files_todo_run{$run}." ${srcdir}/stream$filestream{$type}/$file";
		    $files_tomark_run{$run} = $files_tomark_run{$run}."$file\n";
		    $files_ntodo_run{$run}++;
		    printf("New file $file (run $run [ $drun ] has $files_ntodo_run{$run} new $type files)\n");
		}
	    }
	    else
	    {
#		printf "Skipping file from run $drun < $firstrun\n";
	    }
	}
    } # loop over new files

	# now we have the list of new files for each run, loop over the runs

	my $run;
    my @runstodo = keys %files_todo_run;
    printf("Running over runs: @runstodo...\n");
#    `sleep 5`;
    for $run (keys %files_todo_run)
     {
	 my $drun = hex $run;
	 my $datestring = `date`;
	 chop($datestring);
	 printf("[$datestring] Run: $run [ $drun ]\n");
	 my $newfiles = $files_todo_run{$run};
	 my $cfile = "";

	 # if we have already done files before, make the file name
	 if ($files_done_run{$run} > 0)
          {
	      $cfile = &concatFileName($run,$type,$files_done_run{$run},1);
          }

	 # name of the new file name
	 my $new_cfile = &concatFileName($run,$type,$files_done_run{$run} + $files_ntodo_run{$run},0);
	 my $logfile = &logFileName($run,$type);
	 
	 open(GOUT,">>$logfile");
		    
	 print GOUT (">>> Need files $cfile $newfiles. Concatenating into $new_cfile.\n");
		    
	 # run the concat script 
	 my $cmd = "cd ${scriptdir} ;./concatmany_$fileclass{$type}.sh $new_cfile $params{$type} $cfile $newfiles 2>&1";
	 print GOUT (" Issuing: $cmd\n");
	 my $res = `$cmd`;
	 printf GOUT (" Result: $res\n");
	 close(GOUT);

	 if ($cfile ne "") # && $type ne "pesgain")
	 {
	     $cmd = "rm $cfile";
	     print GOUT (" Issuing: $cmd\n");
	     my $res = `$cmd`;
	     printf GOUT (" Result: $res\n");
	 }
		    
	 # add to the list of done files
	 my $lfile = &listFileName($run,$type);
	 open(LOUT,">>$lfile");
	 printf LOUT "$files_tomark_run{$run}\n";
	 close(LOUT);
#	 printf("Sleeping 5...");
#	 `sleep 5`;
     }

}

&removeLock();
 
printf("Concatenator done.\n");   
	
    
