Skip to content

Commit bc5cfae

Browse files
cmsbuildcmsbuild
authored
Merge pull request #45867 from mmusich/mm_G2GainsValidator_update
Add certificates handling to `G2GainsValidator.py`
2 parents c74b59f + c01d0d2 commit bc5cfae

File tree

4 files changed

+133
-37
lines changed

4 files changed

+133
-37
lines changed

‎CondCore/SiStripPlugins/scripts/G2GainsValidator.py‎

Lines changed: 117 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,20 @@
11
from __future__ import print_function
22

3+
from datetime import datetime
4+
import ROOT
35
import configparser as ConfigParser
6+
import datetime
47
import glob
5-
import os
8+
import json
69
import numpy
10+
import optparse
11+
import os
712
import re
8-
import ROOT
13+
import sqlalchemy
914
import string
1015
import subprocess
1116
import sys
12-
import optparse
1317
import time
14-
import json
15-
import datetime
16-
from datetime import datetime
1718
import CondCore.Utilities.conddblib as conddb
1819

1920
##############################################
@@ -31,26 +32,87 @@ def getCommandOutput(command):
3132
return data
3233

3334
##############################################
34-
def getFCSR():
35+
def getCerts() -> str:
36+
##############################################
37+
cert_path = os.getenv('X509_USER_CERT', '')
38+
key_path = os.getenv('X509_USER_KEY', '')
39+
40+
certs = ""
41+
if cert_path:
42+
certs += f' --cert {cert_path}'
43+
else:
44+
print("No certificate, nor proxy provided for Tier0 access")
45+
if key_path:
46+
certs += f' --key {key_path}'
47+
return certs
48+
49+
##############################################
50+
def build_curl_command(url, proxy="", certs="", timeout=30, retries=3, user_agent="MyUserAgent"):
51+
##############################################
52+
"""Builds the curl command with the appropriate proxy, certs, and options."""
53+
cmd = f'/usr/bin/curl -k -L --user-agent "{user_agent}" '
54+
55+
if proxy:
56+
cmd += f'--proxy {proxy} '
57+
else:
58+
cmd += f'{certs} '
59+
60+
cmd += f'--connect-timeout {timeout} --retry {retries} {url}'
61+
return cmd
62+
3563
##############################################
36-
out = subprocess.check_output(["curl", "-k", "-s", "https://cmsweb.cern.ch/t0wmadatasvc/prod/firstconditionsaferun"])
64+
def get_hlt_fcsr(session):
65+
##############################################
66+
RunInfo = session.get_dbtype(conddb.RunInfo)
67+
lastRun = session.query(sqlalchemy.func.max(RunInfo.run_number)).scalar()
68+
fcsr = lastRun+1
69+
return int(fcsr)
70+
71+
##############################################
72+
def getFCSR(proxy="", certs=""):
73+
##############################################
74+
url = "https://cmsweb.cern.ch/t0wmadatasvc/prod/firstconditionsaferun"
75+
cmd = build_curl_command(url, proxy=proxy, certs=certs)
76+
out = subprocess.check_output(cmd, shell=True)
3777
response = json.loads(out)["result"][0]
3878
return int(response)
3979

4080
##############################################
41-
def getPromptGT():
81+
def getPromptGT(proxy="", certs=""):
4282
##############################################
43-
out = subprocess.check_output(["curl", "-k", "-s", "https://cmsweb.cern.ch/t0wmadatasvc/prod/reco_config"])
83+
url = "https://cmsweb.cern.ch/t0wmadatasvc/prod/reco_config"
84+
cmd = build_curl_command(url, proxy=proxy, certs=certs)
85+
out = subprocess.check_output(cmd, shell=True)
4486
response = json.loads(out)["result"][0]['global_tag']
4587
return response
4688

4789
##############################################
48-
def getExpressGT():
90+
def getExpressGT(proxy="", certs=""):
4991
##############################################
50-
out = subprocess.check_output(["curl", "-k", "-s", "https://cmsweb.cern.ch/t0wmadatasvc/prod/express_config"])
92+
url = "https://cmsweb.cern.ch/t0wmadatasvc/prod/express_config"
93+
cmd = build_curl_command(url, proxy=proxy, certs=certs)
94+
out = subprocess.check_output(cmd, shell=True)
5195
response = json.loads(out)["result"][0]['global_tag']
5296
return response
5397

98+
##############################################
99+
def resetSynchonization(db_name):
100+
##############################################
101+
import sqlite3
102+
103+
# Connect to the SQLite database
104+
conn = sqlite3.connect(db_name)
105+
106+
# Create a cursor object to execute SQL commands
107+
cursor = conn.cursor()
108+
109+
# Execute the SQL command to update the database
110+
cursor.execute("UPDATE TAG SET SYNCHRONIZATION='any' WHERE SYNCHRONIZATION='express';")
111+
112+
# Commit the changes and close the connection
113+
conn.commit()
114+
conn.close()
115+
54116
##############################################
55117
if __name__ == "__main__":
56118
##############################################
@@ -68,17 +130,39 @@ def getExpressGT():
68130
default = -1,
69131
help = 'sinces to copy from validation tag',
70132
)
71-
133+
134+
parser.add_option('-p', '--proxy',
135+
dest = 'proxy',
136+
default = "",
137+
help = 'proxy to use for curl requests',
138+
)
139+
140+
parser.add_option('-u', '--user-mode',
141+
dest='user_mode',
142+
action='store_true',
143+
default=False,
144+
help='Enable user mode with specific X509 user certificate and key')
145+
72146
(options, arguments) = parser.parse_args()
73147

148+
if options.user_mode:
149+
os.environ['X509_USER_KEY'] = os.path.expanduser('~/.globus/userkey.pem')
150+
os.environ['X509_USER_CERT'] = os.path.expanduser('~/.globus/usercert.pem')
151+
print("User mode enabled. Using X509_USER_KEY and X509_USER_CERT from ~/.globus/")
74152

75-
FCSR = getFCSR()
76-
promptGT = getPromptGT()
77-
expressGT = getExpressGT()
78-
print ("Current FCSR:",FCSR,"| Express Global Tag",expressGT,"| Prompt Global Tag",promptGT)
153+
certs = ""
154+
if not options.proxy:
155+
certs = getCerts()
156+
157+
FCSR = getFCSR(proxy=options.proxy, certs=certs)
158+
promptGT = getPromptGT(proxy=options.proxy, certs=certs)
159+
expressGT = getExpressGT(proxy=options.proxy, certs=certs)
79160

80161
con = conddb.connect(url = conddb.make_url("pro"))
81162
session = con.session()
163+
164+
HLTFCSR = get_hlt_fcsr(session)
165+
print ("Current next HLT run", HLTFCSR, "| curret FCSR:", FCSR ,"| Express Global Tag",expressGT,"| Prompt Global Tag",promptGT)
82166
IOV = session.get_dbtype(conddb.IOV)
83167
TAG = session.get_dbtype(conddb.Tag)
84168
GT = session.get_dbtype(conddb.GlobalTag)
@@ -99,12 +183,12 @@ def getExpressGT():
99183
IOVsToValidate=[]
100184
if(options.since==-1):
101185
IOVsToValidate.append(validationTagIOVs[-1][0])
102-
print("changing the default validation tag since to:",IOVsToValidate[0])
186+
print("Changing the default validation tag since to:",IOVsToValidate[0])
103187

104188
else:
105189
for entry in validationTagIOVs:
106190
if(options.since!=1 and int(entry[0])>=int(options.since)):
107-
print("appending to the validation list:",entry[0],entry[1],entry[2])
191+
print("Appending to the validation list:",entry[0],entry[1],entry[2])
108192
IOVsToValidate.append(entry[0])
109193

110194
for element in myGTMap:
@@ -114,24 +198,33 @@ def getExpressGT():
114198
Tag = element[2]
115199
if(Record=="SiStripApvGain2Rcd"):
116200
TagIOVs = session.query(IOV.since,IOV.payload_hash,IOV.insertion_time).filter(IOV.tag_name == Tag).all()
117-
lastG2Payload = TagIOVs[-1]
118-
print("last payload has IOV since:",lastG2Payload[0],"payload hash:",lastG2Payload[1],"insertion time:",lastG2Payload[2])
201+
sorted_TagIOVs = sorted(TagIOVs, key=lambda x: x[0])
202+
lastG2Payload = sorted_TagIOVs[-1]
203+
print("Last G2 Prompt payload has IOV since:",lastG2Payload[0],"payload hash:",lastG2Payload[1],"insertion time:",lastG2Payload[2])
204+
205+
# Get and print the current working directory
206+
current_directory = os.getcwd()
207+
print("Current Working Directory:", current_directory)
208+
119209
command = 'conddb_import -c sqlite_file:toCompare.db -f frontier://FrontierProd/CMS_CONDITIONS -i '+str(Tag) +' -t '+str(Tag)+' -b '+str(lastG2Payload[0])
120210
print(command)
121211
getCommandOutput(command)
122212

213+
# set syncrhonization to any
214+
resetSynchonization("toCompare.db")
215+
123216
for i,theValidationTagSince in enumerate(IOVsToValidate):
124217

125218
command = 'conddb_import -c sqlite_file:toCompare.db -f frontier://FrontierPrep/CMS_CONDITIONS -i '+str(options.validationTag) +' -t '+str(Tag)+' -b '+str(theValidationTagSince)
126219
if(theValidationTagSince < lastG2Payload[0]):
127-
print("the last available IOV in the validation tag is older than the current last express IOV, taking FCSR as a since!")
220+
print("The last available IOV in the validation tag is older than the current last express IOV, taking FCSR as a since!")
128221
command = 'conddb_import -c sqlite_file:toCompare.db -f frontier://FrontierPrep/CMS_CONDITIONS -i '+str(options.validationTag) +' -t '+str(Tag)+' -b '+str(FCSR+i)
129222

130223
print(command)
131224
getCommandOutput(command)
132225

133-
command = './testCompare.sh SiStripApvGain_FromParticles_GR10_v1_express '+str(lastG2Payload[0])+' '+str(theValidationTagSince)+ ' toCompare.db'
226+
command = '${CMSSW_BASE}/src/CondCore/SiStripPlugins/scripts/testCompare.sh SiStripApvGain_FromParticles_GR10_v1_express '+str(lastG2Payload[0])+' '+str(theValidationTagSince)+' '+current_directory+'/toCompare.db'
134227
if(theValidationTagSince < lastG2Payload[0]):
135-
command = './testCompare.sh SiStripApvGain_FromParticles_GR10_v1_express '+str(lastG2Payload[0])+' '+str(FCSR+i)+ ' toCompare.db'
228+
command = '${CMSSW_BASE}/src/CondCore/SiStripPlugins/scripts/testCompare.sh SiStripApvGain_FromParticles_GR10_v1_express '+str(lastG2Payload[0])+' '+str(FCSR+i)+' '+current_directory+'/toCompare.db'
136229
print(command)
137230
getCommandOutput(command)

‎CondCore/SiStripPlugins/scripts/testCompare.sh‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,25 @@ display_usage() {
66
}
77

88
# if less than two arguments supplied, display usage
9-
if [ $# -le 3 ]
10-
then
11-
display_usage
12-
exit 1
13-
fi
9+
if [ $# -le 3 ]
10+
then
11+
display_usage
12+
exit 1
13+
fi
1414

1515
# check whether user had supplied -h or --help . If yes display usage
16-
if [[ ( $# == "--help") || $# == "-h" ]]
17-
then
18-
display_usage
19-
exit 0
20-
fi
16+
if [[ ( $# == "--help") || $# == "-h" ]]
17+
then
18+
display_usage
19+
exit 0
20+
fi
2121

2222
# Save current working dir so img can be outputted there later
2323
W_DIR=$(pwd);
24-
# Set SCRAM architecture var
25-
SCRAM_ARCH=slc6_amd64_gcc630;
24+
2625
STARTIOV=$2
2726
ENDIOV=$3
2827

29-
export SCRAM_ARCH;
3028
source /afs/cern.ch/cms/cmsset_default.sh;
3129
eval `scram run -sh`;
3230
# Go back to original working directory

‎CondCore/SiStripPlugins/test/BuildFile.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@
88
<use name="CalibTracker/SiStripCommon"/>
99
<bin file="testSiStripPayloadInspector.cpp" name="testSiStripPayloadInspector">
1010
</bin>
11+
<test name="test_G2GainsValidator" command="test_G2GainsValidator.sh"/>

‎CondCore/SiStripPlugins/test/test_G2GainsValidator.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
3+
function die { echo $1: status $2 ; exit $2; }
4+
python3 $CMSSW_BASE/src/CondCore/SiStripPlugins/scripts/G2GainsValidator.py || die "Failure running G2GainsValidator.py" $?

0 commit comments

Comments
 (0)