#! /bin/tcsh -f # ----------------------------------------------------------------------------- # # Name : merge_svxalign.tcsh # # Purpose : Merge the PROD_PHYSICS_SVXALIGN used_set # into the PROD_PHYSICS_CDF used_set with the matching run number. # # Usage : merge_svxalign.tcsh # # Options : None. # # Arguements : None. # # Created : wed May 8 2002 Copied from merge_svx.tcsh # # Updated : 24/7/02 add robs database and pid checks # # Updated : comment out active database-write code until confirm # that it is working properly under cdfcalib # # Updated : 6/8/02 added filling of pass 03 and fixed began working sucessfully under cdfcalib # Updated : 4/9/02 Rob Snihur Fill pass tables if v4 (instead of v16) exists. # # Updated : 10/10/02 Dustin McGivern: added filling of pass06 after change to L3 svxalign valid set # # Updated : 18/10/02 Dustin McGivern: change the version of DBSetMerge being used to Rob's private # version, since the development version is broken. # # Updated : 24/10/02 Dustin McGivern: change script to stop adding svxalign(not needed anymore) and to add # PROD_PHYSICS_CAL jobset 595028. which contains all calorimeter # calibs needed. # # Updated : 8/11/02 Dustin McGivern : add cid checking before filling the passtable. # compare to are those in version 4 of PROD_PHYSCIS_CDF for run 154046; jobset = 610532 # # Updated : 19/11/02 Dustin McGivern : add filling of pass07 # # Updated : 19/11/02 Dustin McGivern : new COT table; so change comparison jobset for cid checking to # 154232 jobset 618777 # # Updated : 19/11/02 Dustin McGivern : COTPROBCH table added to list of tables to ignore. # # Updated : 02/03 Dustin McGivern : change prod_physics_cal valid set to include new tables: 680801 # # Updated : 12/03/03 Dustin McGivern : found problem, typo in valid set : 680801. # this has broken all post-shutdown data!!!! bollocks! # # Updated : 13/03/03 Dustin McGivern : changes in smx tables and whalineresponse since shut down. # updated reference run to 160149, jobset = 709863 # # Updated : 25/4/03 Dustin McGivern : changes in calor tables: new effeicieny and slewing tables. # New refernece used_set: run 160541, version 4 = jobset 794203 # # Updated : 10/5/03 Rob Snihur : New tables in L1_PHYSICS_FWD , & updated table CALDIGITOGEV3 # in L1_PHYSICS_CAL. All these changes inherited from L3_PHYSICS_CDF. # New reference used_set: run 162751, version 4 = jobset 827000. # # # Updated : 12/5/03 Dustin McGivern : Beate has created a new Prod_physics_cal used set to be picked up by L3 # this seems to be being done reliably now, so switch off my merging below. # And change if ver = 3 in pass table section. # # Updated : 14/5/03 Dustin McGivern : New prod_physics_cal valid set above has new chaoffler table. # Update the reference run to take account of this: # ref: run 162904 version 3 jobset = 827638. # # Updated : 27/5/03 Dustin McGivern : Major overhaul. bound to screw something up # New alignment tables needed for 4.10.4 production. But must keep 4.8.4 # production alignment tables as they are. here's the plan: # 1) merge in new valid set. # 2) check cids of *previous* version(the one for 4.8.4). # if good then fill passtables # => use $highest_version -1 since the new current highest is for 4.10.4 . # 3) check cids of highest version aganst a pass08 reference run. # if good fill pass08 using highest_version. # # Pass08 Reference run = 162686 version 4 jobset = 829384 # # Updated : 28/05/03 Dustin McGivern : Change in calorimeter calibration tables. need to update reference runs # For pass02-07 new reference is version 3 of run 163650 jobset 834500 # For pass08 new reference is version 4 of run 163650 jobset 834508 # # Updated : 11/6/03 Rob Snihur : Check for numeric runnumber; add exit 4. # # Updated : 13/6/03 Dustin McGivern : Change in calorimeter tables, update reference runs again: # For pass02-07 new reference is version 3 of run 164651 jobset 892588 # For pass08 new reference is version 4 of run 164651 jobset 892594 # # Updated : 14/07/03 Dustin McGivern : added extra line to list of text to be ignored by error checking: # grep -v "dcap" to suppress errors caused by setup cdfsoft2 command. # # Updated : 16/07/03 Dustin McGivern : update rference runs due to new pprlineresponse table. # For pass02-07 new reference is version 3 of run 166088 jobset 898361 # For pass08 new reference is version 4 of run 166088 jobset 898365 # # Updated : 21/08/03 Dustin McGivern : updated for new cal tables. # For pass02-07 new reference is version 3 of run 167824 jobset 1026653 # For pass08 new reference is version 4 of run 167824 jobset 1026655 # # Updated : 2/09/03 Dustin McGivern : updated for new tof tables # For pass02-07 new reference is version 3 of run 168439 jobset 1029398 # For pass08 new reference is version 4 of run 168439 jobset 1029406 # # Updated : 4/09/03 Dustin McGivern : updated for even more new tof tables # For pass02-07 new reference is version 3 of run 168812 jobset 1030966 # For pass08 new reference is version 4 of run 168812 jobset 1031266 # # Updated : a while ago Dustin+Ben : moved script to new area and renamed it fill_passtable.tcsh # # Updated : 26/11/03 Dustin McGivern : discontinuing 4.8.4 and 4.10.4 passtables. Only need to fill pass10 # need to merge in the 5.1 svxalign valid set since current highest jobset # of PROD_PHYSICS_SVXALIGN(ie that collected by L3) is for 4.8.4 # # Updated : 30/11/03 Dustin McGivern : adding the new L3_PHYSICS_SMX valid_set since expert screwed up # and missed them out of the new data runs. Also adding PROD_PATCH_CAL # used_set until the calorimeter group sort out the PROD_PHYSICS_CAL # valid set, which is currently out of date. # Also, updating the golden run to jobset 1144825 a specially made # prod_physics_cdf used_set for run 175092 # #Updated : 1/12/03 Dustin McGivern : adding L1_PHYSICS_CAL to prevent errors due to L3 not picking up # the latest valid_set of L1_PHYSICS_CAL when producing L3_PHYSICS_CDF # Update the golden run to account for this. # #Updated : 09/12/03 Ben Cooper : Switched to cdfsoftware 4.11.2 after 4.11.1 was discontinued. # #Updated : 10/12/03 Ben Cooper : Added in addititonal error checking to make sure that PROD_PHYSICS_SVX/COT # have definately been merged before pass 10 is filled. This is done by # comparing PROD_PHYSICS_CDF version 4 jobset with the jobset of # PROD_PHYSICS_COT/SVX on a run by run basis. # #Updated :16/12/03 Ben Cooper : After Willis added new BADCHANNEL tables into L3_PHYSICS_CAL we need to # update the golden run jobset - choose version 4 176422 PROD_PHYSICS_CDF. # #Updated :15/01/04 Ben Cooper : Removed the merging of fixed SMX (1144617) and L1(1142195) valid sets as this # was only a temporary fix for just after the Oct-Nov 2003 shutdown. Now these # tables are automatically picked up by L3. # #Updated :18/01/04 Ben Cooper : Removal of merging of fixed valid sets explained above means that golden reference # run needs to be updated to a run which has the new tables inherited from L3. Choose # 177940 (version 4 jobset 1176931) # #Updated :19/01/04 Ben Cooper : Removal of merging of fixed CAL 1036025 jobset as this was a temporary patch and the # correct tables are now inherited by L3 (as advised by Willis). Necessitates the changing # of the golden reference run again. Choose 178015 (version 4 jobset 1177185). #Updated :25/01/04 Ben Cooper : Comparison of PROD_PHYSICS_CDF COT tables versus those in used_set improved - now only # performs this check if the COT table has evaluation!='UNDEFINED'. If COT table is undefined # then the used_set is not merged into PROD_PHYSICS_CDF and thus these runs would always fail # this test. #Updated :28/03/04 Ben Cooper : Script changed to fill_passtable_10A - now merge in the 5.3.1 alignment valid_set and # fills pass 10A. As we are now merging in a different alignment valid_set we also have # to update the golden reference run to one which uses thid valid_set. Choose 180958. #Updated :14/04/04 Ben Cooper : Calorimeter tables inherited by L3 are changed, thus we need to update the golden reference run # Choose 181632 (first run after calorimeter table changes). #Updated :14/06/04 Ben Cooper : TOF tables are changed - have to update golden ref. run to 183868 # --------------------------------------------------------------------------------------------------------- #look for any other processes with the same name and exit if you find them #(we don't want multiple copies of the same script) ps -fN -p $$ | grep $USER | grep $0 | grep -v grep && exit 5 #setup the software source ~cdfsoft/cdf2.cshrc setup cdfsoft2 4.11.2 set aaaaa_oracle_dir = `echo $ORACLE_CLIENT_DIR` setup cdfdb #is the database working? if not exit with no output dbPing -dcdfonprd -s > /dev/null || exit 6 #move to the right place cd /data1/cdfcalib/cron_scripts/fill_passtable/ #set the jobset of the PROD_PHYSICS_SVXALIGN valid_set to be added @ jobset = 1201303 #@ jobsetSMX = 1144617 #@ jobsetCAL = 1036025 #@ jobsetL1 = 1142195 #look for any new runs @ last_merged_date = `grep . merged_run_and_date.txt | tail -1 | awk '{print int($1)}'` # Get the last date. if ( $last_merged_date == 0 ) then # If run number is invalid. echo "file merged_run_and_date.txt:"; echo "============"; cat merged_run_and_date.txt; exit 2 # Print file, and exit. endif # if ( $last_merged_date == 0 ) then #find any new runs at v3 of prod_physics_cdf sqlplus -s cdf_reader/reader@cdfonprd @svxalign.sql $last_merged_date || exit 3 if (! -z merged_run_and_date.txt) then #there are new runs #==================================== #add in 5.1.x svxalign valid set #==================================== #mark the start of the output so we can find it in the cron output file echo "=============================================new run====================================" set dat = `date` echo "Script ran at $dat" #get the list of runs from the sql ouput cut -d" " -f3- merged_run_and_date.txt >! runs.lst # Check that all runs in list are numeric. foreach r ( `grep . runs.lst` ) # Loop over runs. @ n = 0 # Default. @ n = `echo $r | awk '{print int($1)}'` # Numeric run. if ( $n == 0 ) then # If non-numeric run translated to zero. echo "Error: Exit because of non-numeric run $r" echo "Will not process any runs." echo "file merged_run_and_date.txt:"; echo "============"; cat merged_run_and_date.txt # Print file. echo $last_merged_date >! merged_run_and_date.txt # Put the last processed run in there. exit 4 # Exit. endif # if ( $n == 0 ) end # foreach r ( `grep . runs.lst` ) #loop over runs foreach r (`grep . runs.lst`) echo "in loop" echo "run = $r" @ tof_jobset = `sqlplus -s cdf_reader/reader@cdfonprd @get_max_jobset.sql $r PROD_PHYSICS_TOF` echo "tof_jobset: $tof_jobset" #now make version4 of PROD_PHYSICS_CDF merging in the new svxalign valid set for 5.1.x echo "/cdf/people2/snihur/DBSetMerge.exe/DBSetMerge.20020111 TESTMODE OTL /@cdfonprd used $r PROD_PHYSICS_CDF PROD_PHYSICS_CDF $jobset $tof_jobset" /cdf/people2/snihur/DBSetMerge.exe/DBSetMerge.20020111 TESTMODE OTL /@cdfonprd used $r PROD_PHYSICS_CDF PROD_PHYSICS_CDF $jobset $tof_jobset >& db_testmode_out cat db_testmode_out rm db_testmode_out /cdf/people2/snihur/DBSetMerge.exe/DBSetMerge.20020111 OTL /@cdfonprd used $r PROD_PHYSICS_CDF PROD_PHYSICS_CDF $jobset $tof_jobset >& db_out cat db_out rm db_out end echo "finished making new versions" #=============================================================================================== #------------------------------------------------------------------------------------------------ #Automated filling of passcalibs table # #take this list of runs and fill v4 into passtable #----------------------------------------------------------------------------------------------- rm passcalibs.dat >& /dev/null rm passcalibs_10.dat >& /dev/null echo "removed versions file and passcalib files" #==============================Get pass index for each passtable===================== #pass10 #====== @ highest_pass_index_of_pass_10A = `$CDFSOFT2_DIR/CalibDB/bin/PassTool.pl -c /@cdfonprd -n 10A -l | grep "10A" | grep "PROD_PHYSICS_CDF" | grep -v "YES" | grep "A" | awk '{print $1}' ` echo "got highest pass index for pass 10A: $highest_pass_index_of_pass_10A" #now begin making pass10 input file #echo "P_INDEX LORUN HIRUN PROC_CALIB_VERSION" > passcalibs_10.dat #echo "--------------------------------------------------------------------------" >> passcalibs_10.dat echo " made passcalib file headers" #loop over the runs in the file and make sure they are now at v4 foreach r (`grep . runs.lst`) rm verOfThisRun.txt >& /dev/null #clear file #query the database and pipe the output to a file sqlplus -s cdf_reader/reader@cdfonprd @version_from_run_and_name.sql $r PROD_PHYSICS_CDF > verOfThisRun.txt #assign the parsed file to a variable @ version_of_this_run = `tail -1 verOfThisRun.txt | tr -d "[:blank:]"` #is it at version 4? if ( $version_of_this_run == 4 ) then #fill the passcalibs.dat file if ($r > 168890) then #and add the run number and version number to the passcalibs.dat file echo "$highest_pass_index_of_pass_10A $r $r $version_of_this_run" >> passcalibs_10.dat endif #end if run>shutdown run endif #end if ver = 4 end #end loop over runs in these_runs.out file echo "filled passcalibs file" #================================================================================================== # #Before filling the passtable check the cids of the versions we've just made to see if everything is ok. # #if yes then fill pass table; if no then email Dustin and Rob to say something has changed. # # #================================================================================================== #use robs script which loops over a list if runs and returns the cid's that are different for each run. #seems to need this: #setup cdfsoft2 development #the calib tables we shall compare to are: #for the 5.3.1 calibs(ver4 of PROD_PHYSICS_CDF) compare to those in version 4 of PROD_PHYSCIS_CDF for run 183868; @ good_jobset_5_1 = 1297121 #version 4 of run 183868 echo "reference jobset for ver4 5.1.x = $good_jobset_5_1" #first clean all the temp files we will use: # if(-f cid_changes.out) rm cid_changes.out # if(-f table_changes) rm table_changes # if(-f changes.out) rm changes.out setenv ORACLE_CLIENT_DIR $aaaaa_oracle_dir #run robs script to get the cid differences, first for ver3 and then ver4 ./compare_tableids_jobset_runlist.tcsh $good_jobset_5_1 runs.lst PROD_PHYSICS_CDF 4 >& cid_changes_ver4.out echo "running compare tableids script" cat cid_changes_ver4.out #==========================check cids of ver4======================= #now look through the output file for changes; but first remove the lines refering to tables that we expect to change. grep . cid_changes_ver4.out | grep -v COTCHANT0 | grep -v COTDEADCH | grep -v COTSTAGE0 | grep -v SICHIPPED | grep -v SICHIPTHRESH | grep -v SIDPSPARMS | grep -v SISTRIPDH | grep -v SISTRIPMASK | grep -v SICHIPON | grep -v COTPROBCH | grep -v SVXBEAMPOSITION | grep -v COTBEAMPOSITION | grep -v dcap >& table_changes_ver4 #now remove the header infomation and look for any changes grep . table_changes_ver4 | grep -v "comm -3" | grep -v Number | grep -v compare_tableids_jobset_runlist.tcsh | grep -v dcap >& changes_ver4.out #now if table changes then this file will not be of zero length; so email it to us and exit the script. if (! -z changes_ver4.out) then mail -s "In fill_passtables: Changes in Calibration tables for ver4" -c djm@fnal.gov bdc@hep.ucl.ac.uk snihur@fnal.gov < table_changes_ver4; exit 21 endif #in addition check that the svx/cot tables in PROD_PHYSICS_CDF version 4 agree with the PROD_PHYSICS_SVX/COT jobset for that run #to avoid errors caused by a run reaching a high enough version to progress to this fill_passtable script without having #had svx merged and/or cot merged. foreach run(`grep . runs.lst`) rm svx_cid_changes.out >& /dev/null rm cot_cid_changes.out >& /dev/null #first find PROD_PHYSICS_SVX jobset for that run @ svx_jobset = `sqlplus -s cdf_reader/reader@cdfonprd @get_max_jobset.sql $run PROD_PHYSICS_SVX` ./compare_tableids_jobset_onerun.tcsh $svx_jobset $run PROD_PHYSICS_CDF 4 >& svx_cid_changes.out #counting the number of SI tables in the compare_tableids_jobset_onerun.tcsh output. If there are tables present it #means differences have been found - script exits if ( `grep -c SICHIPON svx_cid_changes.out` || `grep -c SICHIPPED svx_cid_changes.out` || `grep -c SIFIBOFFSETS svx_cid_changes.out` || `grep -c SISTRIPDH svx_cid_changes.out` || `grep -c SISTRIPMASK svx_cid_changes.out` ) then echo "Problem with SVX tables for ver4 - script exiting" cat svx_cid_changes.out; mail -s "In fill_passtables: Problem with SVX tables for ver4" -c djm@fnal.gov bdc@hep.ucl.ac.uk snihur@fnal.gov < svx_cid_changes.out; exit 22 endif #check that $run has a properly defined (i.e. not BAD) COT table otherwise the check below will not work. @ cot_defined = `sqlplus -s cdf_reader/reader@cdfonprd @def_mod.sql $run` #if the COT table is defined $cot_defined will equal the run number, otherwise will be zero #occasionally more than one cot jobset is created by stage_0 - causes problems with the compare jobset test when we try and compare #PROD_PHYSICS_CDF with the max COT jobset which wasn't actually that merged in. Quick fix - if the number of COT jobsets isn't equal #to zero don't bother with the test. @ num_cot_jobsets = `sqlplus -s cdf_reader/reader@cdfonprd @get_all_jobsets.sql $run PROD_PHYSICS_COT | grep -c .` @ testable = 1; if(($cot_defined) && (($num_cot_jobsets)==($testable))) then echo "COT table is well defined - checking $run PROD_PHYSICS_CDF COT tables against COT used_set." #counting the number of COT tables in the compare_tableids_jobset_onerun.tcsh output. If there are tables present it #means differences have been found - script exits @ cot_jobset = `sqlplus -s cdf_reader/reader@cdfonprd @get_max_jobset.sql $run PROD_PHYSICS_COT` ./compare_tableids_jobset_onerun.tcsh $cot_jobset $run PROD_PHYSICS_CDF 4 >& cot_cid_changes.out if ( `grep -c COTCHANT0 cot_cid_changes.out` || `grep -c COTCTI cot_cid_changes.out` || `grep -c COTPROBCH cot_cid_changes.out` || `grep -c COTSTAGE0 cot_cid_changes.out`) then echo "Problem with COT tables for ver4 - script exiting" cat cot_cid_changes.out; mail -s "In fill_passtables: Problem with COT tables for ver4" -c djm@fnal.gov bdc@hep.ucl.ac.uk snihur@fnal.gov < cot_cid_changes.out; exit 23 endif endif end #foreach run(`grep . runs.lst`) #otherwise remove the temp files and then go on and fill the passtable echo "no cid changes found, so carry on" rm cid_changes_ver4.out rm table_changes_ver4 rm changes_ver4.out #now fill pass10 cp passcalibs_10.dat passcalibs.dat #and run the pasascalib tool $CDFSOFT2_DIR/CalibDB/bin/PassCalibTool.pl -c /@cdfonprd -s -r passcalibs.dat #to screen for cronjob echo "made insert statements; now filling table" $CDFSOFT2_DIR/CalibDB/bin/PassCalibTool.pl -c /@cdfonprd -r passcalibs.dat #to screen for cronjob echo "filled passcalib table" #if successful then put this passcalibs.dat into a log file for safe keeping and errorchecking. cat passcalibs_10.dat #for cronjob just cat file #remove the tempoary files to save space rm passcalibs_10.dat >& /dev/null rm passcalibs.dat >& /dev/null rm verOfThisRun.txt >& /dev/null #and mark the end of the run in the cron output file echo "==============================end run====================================" else # No new runs detected. echo $last_merged_date >> merged_run_and_date.txt # Put the last processed run in there. endif #end if new runs