#!/usr/bin/env python3 # pylint: disable=C0301 # too long lines # pylint: disable=C0116 # docstring # pylint: disable=C0114 # module docstring # pylint: disable=C0302 # too many lines in the module # pylint: disable=C0305 # trailing newlines # pylint: disable=C0410 # multiple imports in one line # pylint: disable=W0613 # unused argument # pylint: disable=W0603 # using global # pylint: disable=W0703 # too general exception # pylint: disable=W0612 # unused variable # pylint: disable=R0912 # too many branches # pylint: disable=R0911 # too many return statements # pylint: disable=R0915 # too many statements from optparse import OptionParser from urllib import request as urlrequest import sys import datetime import shutil import os, os.path import subprocess import signal import time import pwd import json # Order is important SUPPORTED_COMMANDS = ['all', 'build', 'functional_test', 'stability_test', 'memcheck', 'stress_test', 'compare', 'production_tests', 'perf_test', 'archive'] VERBOSE = False PORT1 = 10999 PORT2 = 10888 SRV1 = None SRV2 = None # Sometimes, especially under valgrind, it takes longer to start. # So, wait a bit before running a test suite AFTER_START_DELAY = 45 AFTER_STOP_DELAY = 45 PRODUCTION_SERVICE = 'PSG2' DEV_SERVICE = 'PSG2_DEV' def timestamp(): now = datetime.datetime.now() return now.strftime('%Y-%m-%d %H:%M:%S') def print_verbose(msg): if VERBOSE: print(timestamp() + ' ' + msg) def print_error(msg): print(timestamp() + ' ' + msg, file=sys.stderr) def getArtefactDir(options): try: dirName = 'PSG-' + options.psg_ver except: raise Exception('Script logic error: options must be checked before proceeding') currentDir = os.path.abspath(os.getcwd()) fullPath = os.path.normpath(currentDir + os.path.sep + dirName) if not fullPath.endswith(os.path.sep): fullPath += os.path.sep return fullPath def safeRemove(filePath): try: if os.path.exists(filePath): os.remove(filePath) except Exception as exc: print_error(f'Error removing file: {filePath}: ' + str(exc)) except: print_error(f'Unknown error removing file: {filePath}') def getPsgClientPath(): try: path = os.environ['NCBI'] path = os.path.normpath(path + os.path.sep + 'bin' + os.path.sep + 'psg_client') if os.path.exists(path): return path return None except Exception as exc: print_error('Error getting path to psg_client: ' + str(exc)) return None def prepareConfigFile(source, target, subst): f = open(source, 'r') content = f.read() f.close() for sub in subst: content = content.replace('{' + sub + '}', subst[sub]) f = open(target, 'w') f.write(content) f.close() def getBuildResultsDir(options): artefactDir = getArtefactDir(options) buildResultsDir = None for item in os.listdir(artefactDir): if item.startswith('prepare_release.'): if item.endswith(options.psg_ver): if '.build.' in item: buildResultsDir = artefactDir + item + os.path.sep break if buildResultsDir is None: print_error('Cannot find the build directory in created on the "build" stage. ' + f'Expected here: {artefactDir}') return None return buildResultsDir def detectFlavor(): if os.path.exists("/etc/almalinux-release"): return "Alma" if os.path.exists("/etc/redhat-release"): return "Centos" raise Exception('Linux flavor detect error. Only Centos and Alma are supported.') def findBuildArchive(buildResultsDir, variant): buildArchive = None flavor = detectFlavor() for item in os.listdir(buildResultsDir): if item.startswith(f'Linux64-{flavor}-pubseq_gateway-'): if item.endswith('.tar.gz'): if not item.endswith('src.tar.gz'): if '-' + variant + '-' in item: buildArchive = buildResultsDir + item break if buildArchive is None: print_error(f'Cannot find the "{variant}" build results in {buildResultsDir}') return None return buildArchive def findBuildResultDir(buildResultsDir, variant): buildResultDir = None flavor = detectFlavor() for item in os.listdir(buildResultsDir): if item.startswith(f'Linux64-{flavor}-pubseq_gateway-'): if os.path.isdir(buildResultsDir + item): if variant in item: buildResultDir = buildResultsDir + item + os.path.sep break if buildResultDir is None: print_error(f'Cannot find the unpacked "{variant}" build results in {buildResultsDir}') return None return buildResultDir def prepareSandboxFiles(buildResultsDir, testDir, artefactDir): # Unpack the source code. Needed items are: # - baseline ini # - LMDB cache # - https cert srcArchive = f'{buildResultsDir}releaseSrc.tar.gz' if not os.path.exists(srcArchive): print_error(f'Cannot find the source code archive {srcArchive}') return 1 cmdLine = f'cd {buildResultsDir} && tar -xzf {srcArchive}' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Unpacking the source code archive return code: {ret}') if ret != 0: print_error(f'Unpacking the source code archive {srcArchive} error') return ret # Copy the baseline ini file, LMDB cache and https files filesToCopy = ['pubseq_gateway.ini.baseline', 'frozen_db_cache/bioseq_info.db', 'frozen_db_cache/blob_prop.db', 'frozen_db_cache/si2csi.db', 'ssl/psg.crt', 'ssl/psg.key'] for fName in filesToCopy: cmdLine = f'cp -f {testDir}{fName} {artefactDir}' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Copy {fName} file return code: {ret}') if ret != 0: print_error(f'Error copying test file {fName} from {testDir}') return ret return 0 def saveProcessArtefacts(srv, dst): content = f'Return code: {srv.returncode}\n\n' if srv.stdout is None: content += 'StdOut:\nNone\n\n' else: content += 'StdOut:\n' + srv.stdout.read().decode('utf-8') + '\n\n' if srv.stderr is None: content += 'StdErr:\nNone' else: content += 'StdErr:\n' + srv.stderr.read().decode('utf-8') f = open(dst, 'w') f.write(content) f.close() def stopServer(srv, reset=True): global SRV1 global SRV2 if srv is SRV1: print_verbose('Sending SIGINT to server 1 ...') srv.send_signal(signal.SIGINT) srv.wait() if reset: SRV1 = None time.sleep(AFTER_STOP_DELAY) else: print_verbose('Sending SIGINT to server 2 ...') srv.send_signal(signal.SIGINT) srv.wait() if reset: SRV2 = None time.sleep(AFTER_STOP_DELAY) def prepareO2Gsandbox(options): artefactDir = getArtefactDir(options) buildResultsDir = getBuildResultsDir(options) if buildResultsDir is None: return 1 print_verbose(f'Build result directory: {buildResultsDir}') # Find the archive with o2g build o2gArchive = findBuildArchive(buildResultsDir, 'O2g') if o2gArchive is None: return 1 # Need to create a sandbox with the server binary, config file, LMDB cache # Unpack the binary archive cmdLine = f'cd {buildResultsDir} && tar -xzf {o2gArchive}' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Unpacking the build archive return code: {ret}') if ret != 0: print_error(f'Unpacking the build archive {o2gArchive} error') return ret # Find the directory where the build result are unpacked o2gBuildResultDir = findBuildResultDir(buildResultsDir, 'O2g') if o2gBuildResultDir is None: return 1 # Copy the binary cmdLine = f'cp -f {o2gBuildResultDir}bin/pubseq_gateway {artefactDir}' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Copy binary return code: {ret}') if ret != 0: print_error(f'Copy binary {o2gBuildResultDir}bin/pubseq_gateway error') return ret testDir = f'{buildResultsDir}c++/src/app/pubseq_gateway/server/test/' # Copy LMDB, ini, https files if prepareSandboxFiles(buildResultsDir, testDir, artefactDir) != 0: return 1 return 0 def getCPUUsage(srv): import psutil proc = psutil.Process(pid=srv.pid) # CPU percent (float) during the 3 seconds interval (blocking call) return proc.cpu_percent(interval=3) def getConcurrentProcCount(port, name): url = f'http://localhost:{port}/ADMIN/info' req = urlrequest.Request(url) response = urlrequest.urlopen(req, timeout=10) content = response.read().decode('utf-8') info = json.loads(content) if name in info: return info[name] return None def getArchivedBinaryPath(ver): if not ver: print_error('Another PSG binary version must be provided ' 'via the --against-binary option') return None flavor = detectFlavor() candidateDir = f'/am/ncbiapdata/release/pubseq_gateway/builds/{ver}/Linux64/' try: for item in os.listdir(candidateDir): if os.path.isdir(candidateDir + item): if item.startswith(f'Linux64-{flavor}-'): if item.endswith('-O2g'): candidate = candidateDir + item + '/bin/pubseq_gateway' if os.path.exists(candidate): return candidate print_error(f'Another PSG binary is not found. Expected here: {candidate}') return None except Exception as exc: print_error('Error getting another PSG binary of version ' + ver + '\n' + str(exc)) return None print_error(f'Cannot find another PSG binary of version {ver}') return None def resolveService(name): import ncbi.grid.deployment_info.services as services instances = [] deploymentServices = services.get_services() for svc in deploymentServices[0]: if svc.name.upper() == name.upper(): instances.append((svc.host, int(svc.port))) return instances def getDevProdServerPair(): devInstances = resolveService(DEV_SERVICE) prodInstances = resolveService(PRODUCTION_SERVICE) # They must work on the same host for devInstance in devInstances: for prodInstance in prodInstances: if devInstance[0] == prodInstance[0]: return devInstance[0] + ':' + str(devInstance[1]), prodInstance[0] + ':' + str(prodInstance[1]) return None, None def getDevServiceInstanceByVersion(ver): devInstances = resolveService(DEV_SERVICE) if not devInstances: print_error(f'Cannot find dev instance in {DEV_SERVICE}') return None tryInstances = [] for devInstance in devInstances: instance = devInstance[0] + ':' + str(devInstance[1]) tryInstances.append(instance) # Get version try: print_verbose(f'Getting PSG version from {instance}') url = f'http://{instance}/ADMIN/info' req = urlrequest.Request(url) response = urlrequest.urlopen(req, timeout=10) content = response.read().decode('utf-8') info = json.loads(content) instanceVer = info["Version"] if instanceVer == ver: return instance except Exception as exc: err = str(exc) print_verbose(f'Error getting version from {instance}: {err}') # Here: no suitable instance was found instances = ', '.join(tryInstances) print_error(f'Could not find a PSG instance of version {ver} in service {DEV_SERVICE}') return None def getArchivedTestPath(options): binPath = getArchivedBinaryPath(options.psg_ver) if binPath is None: return None # strip '/bin/pubseq_gatewa' testsPath = os.path.dirname(binPath) testsPath = os.path.dirname(testsPath) testsPath += os.path.sep + 'tests' + os.path.sep if not os.path.exists(testsPath): print_error(f'Cannot find the archived release tests directory. Expected here: {testsPath}') return None return testsPath def getPerfTestResultsPath(options): # it is like results.20211221-09360 artefactDir = getArtefactDir(options) for item in os.listdir(artefactDir): if item.startswith('results.'): candidate = artefactDir + item + os.path.sep item = item.replace('results.', '') if item[0].isdigit() and item[-1].isdigit() and '-' in item: if os.path.isdir(candidate): return candidate print_error(f"Cannot find the perf_test results directory. Expected at: {artefactDir} as 'results.dddddddd-ddddd'") return None def main(): global VERBOSE parser = OptionParser( """ %prog [options] Supported commands: no command - print help message build - tag, configure, build and archive the release (accepts all arguments of the prepare_release script) functional_test - run functional tests stability_test - run tests when many clients drop their connections memcheck - run tests against 'sanitized' build stress_test - run stress tests compare - compare performance against another release production_tests - run smoke tests against production PSG2 service perf_test - run performance tests archive - Archive the completed release all - do all of the above except archive """) parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="be verbose (default: False)") parser.add_option("--psg-version", action="store", type="string", dest="psg_ver", help="PSG release: should be int.int.int") parser.add_option("--against-binary", action="store", type="string", dest="against_binary", help="The PSG version against which the performance is " "compared: should be int.int.int") parser.add_option("-g", "--debug", action="store_true", dest="debug", default=False, help="don't clean up garbage after failed run (default: False)") parser.add_option("-m", "--message", default="NOJIRA", dest="message", help="SVN commit message (default: NOJIRA). " "Affects 'tag', 'conf' and 'build' actions " "if --local option is not given.") options, args = parser.parse_args() if len(args) == 0: parser.print_help() return 0 if len(args) != 1: print_error('Bad arguments. Command is not specified') return 1 command = args[0] if command not in SUPPORTED_COMMANDS + ['all']: print_error(f'Command "{command}" is not supported') return 1 if command == 'all': commands = SUPPORTED_COMMANDS commands.remove('archive') commands.remove('perf_test') else: commands = [command] VERBOSE = options.verbose # The commands are in the order of the execution # Loop 1: check the command arguments before starting for command in commands: if command == 'all': continue checkArgsMethod = globals().get(f'{command}_check_args') if checkArgsMethod: print_verbose(f'Invoking check arguments for command "{command}"...') ret = checkArgsMethod(commands, options, args) if ret != 0: print_verbose(f'Failed to check arguments for command "{command}"') return ret else: print_verbose(f'Check arguments method for command "{command}" is not found. Skipping.') # Loop 2: check the command implementation for command in commands: if command == 'all': continue method = globals().get(command) if method: print_verbose(f'Implementation of the command "{command}" is found') else: print_error(f'Implementation of the command "{command}" is not found') return 1 # Loop 3: invoking the commands for command in commands: if command == 'all': continue method = globals().get(command) print_verbose(f'Invoking command "{command}"...') ret = method(commands, options, args) print_verbose(f'Command "{command}" finished with return code {ret}') if ret != 0: return ret return 0 def checkVersionOptionAndDirectory(options): if not options.psg_ver: print_error('The "--psg-version" option must be specified') return 1 artefactDir = getArtefactDir(options) if not os.path.exists(artefactDir): print_error(f'The build stage directory {artefactDir} is not found') return 1 return 0 # Set of methods which should check the options for the command # The names must be formed as {command}_check_args def build_check_args(commands, options, args): if shutil.which('prepare_release') is None: print_error('prepare_release executable is not in the PATH') return 1 if not options.psg_ver: print_error('The "--psg-version" option must be specified') return 1 if not options.message: print_error('The "--message" option must be specified') return 1 artefactDir = getArtefactDir(options) if os.path.exists(artefactDir): print_error(f'The directory for storing results {artefactDir} already exists. ' 'Remove it first or use another PSG version') return 1 return 0 def functional_test_check_args(commands, options, args): if 'all' in commands: # Everything will be created on the previous stages return 0 return checkVersionOptionAndDirectory(options) def stability_test_check_args(commands, options, args): if shutil.which('h2load') is None: print_error('h2load executable is not in the PATH') return 1 try: # Needs only for the 'stability_test' action import psutil except: print_error("The 'stability_test' action requires the psutil module to be installed") return 1 if 'all' in commands: # Everything will be created on the previous stages return 0 return checkVersionOptionAndDirectory(options) def memcheck_check_args(commands, options, args): if 'all' in commands: # Everything will be created on the previous stages return 0 return checkVersionOptionAndDirectory(options) def stress_test_check_args(commands, options, args): if shutil.which('h2load') is None: print_error('h2load executable is not in the PATH') return 1 if 'all' in commands: # Everything will be created on the previous stages return 0 return checkVersionOptionAndDirectory(options) def compare_check_args(commands, options, args): if shutil.which('h2load') is None: print_error('h2load executable is not in the PATH') return 1 if not options.against_binary: print_error('The "--against-binary" option must be specified') return 1 if 'all' in commands: # Everything will be created on the previous stages return 0 return checkVersionOptionAndDirectory(options) def production_tests_check_args(commands, options, args): if not options.psg_ver: print_error('The "--psg-version" option must be specified') return 1 try: # it is used only for the perf_test import ncbi.grid.deployment_info.services as services import ncbi.grid.deployment_info.parser as parser except: print_error("The 'production_tests' action requires grid_deployment module to be installed.") return 1 # Check the presence of psg_client, psg_client.py and testcases.json psg_client_path = parser.get_psg_client_path() psg_client_py_path = os.path.dirname(parser.get_psg_client_path()) + os.path.sep + 'psg_client.py' testcases_json_path = os.path.dirname(parser.get_psg_client_path()) + os.path.sep + 'testcases.json' if not os.path.exists(psg_client_path): print_error(f"The 'production_tests' action requires psg_client. Could not find it here: {psg_client_path}") return 1 if not os.path.exists(psg_client_py_path): print_error(f"The 'production_tests' action requires psg_client.py. Could not find it here: {psg_client_py_path}") return 1 if not os.path.exists(testcases_json_path): print_error(f"The 'production_tests' action requires testcases.json. Could not find it here: {testcases_json_path}") return 1 return 0 def perf_test_check_args(commands, options, args): try: # it is used only for the perf_test import ncbi.grid.deployment_info.services as services except: print_error("The 'perf_test' action requires grid_deployment module to be installed.") return 1 if 'all' in commands: # Everything will be created on the previous stages return 0 return checkVersionOptionAndDirectory(options) def archive_check_args(commands, options, args): userName = pwd.getpwuid(os.getuid()).pw_name if userName != 'coremake': print_error("Archive action requires to be performed on behalf of the 'coremake' user") return 1 if checkVersionOptionAndDirectory(options) != 0: return 1 if getBuildResultsDir(options) is None: return 1 if getPerfTestResultsPath(options) is None: return 1 return 0 # Set of methods for running the commands. # The name must match the command name def build(commands, options, args): artefactDir = getArtefactDir(options) print_verbose(f'Artefact directory: {artefactDir}') # Create the artefact directory - throws an exception in case of a problem os.mkdir(artefactDir) cmdLine = f'cd {artefactDir} && prepare_release tag pubseq_gateway {options.psg_ver} ' cmdLine += f'--strip-sensitive-sources=no -m "{options.message}"' if options.debug: cmdLine += ' -g' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Tagging return code: {ret}') if ret != 0: return ret cmdLine = f'cd {artefactDir} && prepare_release conf pubseq_gateway {options.psg_ver} ' cmdLine += f'-m "{options.message}"' if options.debug: cmdLine += ' -g' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Configuring return code: {ret}') if ret != 0: return ret cmdLine = f'cd {artefactDir} && prepare_release build pubseq_gateway {options.psg_ver} ' cmdLine += f'--no-tests -m "{options.message}"' if options.debug: cmdLine += ' -g' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Building return code: {ret}') return ret def functional_test(commands, options, args): global SRV1 if prepareO2Gsandbox(options) != 0: return 1 artefactDir = getArtefactDir(options) # Round up 1: Prepare the config file for UT (http) prepareConfigFile(artefactDir + 'pubseq_gateway.ini.baseline', artefactDir + 'pubseq_gateway.ini', {'si2csi.db': artefactDir + 'si2csi.db', 'bioseq_info.db': artefactDir +'bioseq_info.db', 'blob_prop.db': artefactDir + 'blob_prop.db', 'port': f'{PORT1}', 'ssl_enable': 'false', 'ssl_cert': '', 'ssl_key': ''}) # Run the server logFile = f'{artefactDir}pubseq_gateway-ut-http.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine) time.sleep(AFTER_START_DELAY) # Run the tests errors = [] buildResultsDir = getBuildResultsDir(options) testDir = f'{buildResultsDir}c++/src/app/pubseq_gateway/server/test/' cmdLine = f'{testDir}run_tests.py localhost:{PORT1} > {artefactDir}ut-http.log 2>&1' print_verbose(f'Running http unit tests: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'http unit tests return code: {ret}') if ret != 0: errors.append(f'Error executing http unit tests. Return code: {ret}') print_verbose('http unit tests failed') else: print_verbose('http unit tests succeeded') stopServer(SRV1) # Round up 2: Prepare the config file for UT (https) prepareConfigFile(artefactDir + 'pubseq_gateway.ini.baseline', artefactDir + 'pubseq_gateway.ini', {'si2csi.db': artefactDir + 'si2csi.db', 'bioseq_info.db': artefactDir +'bioseq_info.db', 'blob_prop.db': artefactDir + 'blob_prop.db', 'port': '10999', 'ssl_enable': 'true', 'ssl_cert': artefactDir + 'psg.crt', 'ssl_key': artefactDir + 'psg.key'}) # Run the server logFile = f'{artefactDir}pubseq_gateway-ut-https.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine) time.sleep(AFTER_START_DELAY) # Run the tests cmdLine = f'{testDir}run_tests.py --https localhost:10999 > {artefactDir}ut-https.log 2>&1' print_verbose(f'Running https unit tests: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'https unit tests return code: {ret}') if ret != 0: errors.append(f'Error executing https unit tests. Return code: {ret}') print_verbose('https unit tests failed') else: print_verbose('https unit tests succeeded') stopServer(SRV1) # Round up 3: integration smoke test using psg_client prepareConfigFile(artefactDir + 'pubseq_gateway.ini.baseline', artefactDir + 'pubseq_gateway.ini', {'si2csi.db': artefactDir + 'si2csi.db', 'bioseq_info.db': artefactDir +'bioseq_info.db', 'blob_prop.db': artefactDir + 'blob_prop.db', 'port': '10999', 'ssl_enable': 'false', 'ssl_cert': '', 'ssl_key': ''}) psg_client_path = getPsgClientPath() if psg_client_path is None: errors.append('Cannot find psg_client. The integration smoke tests are skipped') else: cmdLine = f'cp -f {psg_client_path} {artefactDir}' print_verbose(f'Copying psg_client from {psg_client_path} ...') ret = os.system(cmdLine) if ret != 0: errors.append(f'Error copying psg_client from {psg_client_path}') else: # Run the server logFile = f'{artefactDir}pubseq_gateway-integration-test.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine) time.sleep(AFTER_START_DELAY) # Run the tests integrationTestDir = f'{buildResultsDir}c++/src/app/pubseq_gateway/server/integrationsmoketest/' cmdLine = f'{integrationTestDir}integration_test -psgclient {artefactDir}psg_client localhost:{10999} > {artefactDir}integration-test.log 2>&1' print_verbose(f'Running integration tests: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'integration tests return code: {ret}') if ret != 0: errors.append(f'Error executing integration tests. Return code: {ret}') print_verbose('integration tests failed') else: print_verbose('integration tests succeeded') stopServer(SRV1) if errors: print_error('Unit test errors:') for err in errors: print_error(err) return 1 return 0 def stability_test(commands, options, args): global SRV1 if prepareO2Gsandbox(options) != 0: return 1 artefactDir = getArtefactDir(options) # Round up 1: Prepare the config file for UT (http) prepareConfigFile(artefactDir + 'pubseq_gateway.ini.baseline', artefactDir + 'pubseq_gateway.ini', {'si2csi.db': artefactDir + 'si2csi.db', 'bioseq_info.db': artefactDir +'bioseq_info.db', 'blob_prop.db': artefactDir + 'blob_prop.db', 'port': f'{PORT1}', 'ssl_enable': 'false', 'ssl_cert': '', 'ssl_key': ''}) errors = [] testCases = ['ID/resolve?psg_protocol=yes&seq_id=XP_015453951&fmt=json&all_info=yes', 'ID/get_na?fmt=json&all_info=yes&seq_id=NW_017890465&psg_protocol=yes&names=NA000122.1', 'ID/get?seq_id=164040&seq_id_type=3&exclude_blobs=1.1,4.8091,3.3', 'ID/getblob?blob_id=4.509567&tse=whole', 'ID/get_tse_chunk?id2_chunk=999999999&id2_info=25.116773935.5'] cnt = 0 for testCase in testCases: cnt += 1 # Run the server logFile = f'{artefactDir}pubseq_gateway-drop-client-http-cpu-{cnt}.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine) time.sleep(AFTER_START_DELAY) # Run the clients clients = [] for client_number in range(100): clientCmdLine = ['h2load', '-n', '10000', '-c', '4', '-t', '4', '-m', '4', f'http://localhost:{PORT1}/{testCase}'] print_verbose(f'Runnig client #{client_number}: ' + ' '.join(clientCmdLine)) clients.append(subprocess.Popen(clientCmdLine, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)) # Let clients work for 3 seconds time.sleep(3) # Kill the clients print_verbose('Killing all h2load at once...') os.system('killall h2load') for client in clients: client.wait() # Let the server to calm down print_verbose('All h2load have been killed. ' 'Allow the server some time to handle the closed connections.') time.sleep(3) # Check CPU usage cpuUsage = getCPUUsage(SRV1) print_verbose(f'PSG CPU usage for the last few seconds is {cpuUsage}') if cpuUsage > 10.0: errors.append(f'Server CPU usage after clients are killed is too high: {cpuUsage}. Test case: {testCase}') # Stop the server stopServer(SRV1) testCases = [('ID/get?seq_id=164040&seq_id_type=3&trace=no&enable_processor=cassandra&disable_processor=OSG', 'ConcurrentProcCount_CASSANDRA'), ('ID/get?seq_id=JRAS01000001&disable_processor=osg&disable_processor=cassandra', 'ConcurrentProcCount_WGS'), ('ID/get?seq_id=JRAS01000001&disable_processor=osg&disable_processor=wgs', 'ConcurrentProcCount_CASSANDRA'), ('ID/get_na?fmt=json&all_info=yes&seq_id=NG_011231.1&names=SNP', 'ConcurrentProcCount_SNP'), ('ID/get_na?fmt=json&all_info=yes&seq_id=CAA42669.1&names=CDD', 'ConcurrentProcCount_CDD')] cnt = 0 for testCase in testCases: cnt += 1 # Run the server logFile = f'{artefactDir}pubseq_gateway-drop-client-http-proccount-{cnt}.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine) time.sleep(AFTER_START_DELAY) # Run the clients clients = [] for client_number in range(100): clientCmdLine = ['h2load', '-n', '10000', '-c', '4', '-t', '4', '-m', '4', f'http://localhost:{PORT1}/{testCase[0]}'] print_verbose(f'Runnig client #{client_number}: ' + ' '.join(clientCmdLine)) clients.append(subprocess.Popen(clientCmdLine, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)) # Let clients work for 3 seconds time.sleep(3) # Kill the clients print_verbose('Killing all h2load at once...') os.system('killall h2load') for client in clients: client.wait() # Let the server to calm down print_verbose('All h2load have been killed. ' 'Allow the server some time to handle the closed connections.') time.sleep(3) try: # Check the number of processors procCount = getConcurrentProcCount(PORT1, testCase[1]) print_verbose(f'PSG processor count after killing clients: {testCase[1]}={procCount}') if procCount != 0: errors.append(f'Processor count after killing clients is not 0. {testCase[1]}={procCount}') except Exception as exc: errors.append(f'Error getting the {testCase[1]} counter: ' + str(exc)) except: errors.append(f'Unknown error getting the {testCase[1]} counter') # Stop the server stopServer(SRV1) if errors: print_error('Drop client test errors:') for err in errors: print_error(err) return 1 return 0 def memcheck(commands, options, args): global SRV1 artefactDir = getArtefactDir(options) buildResultsDir = getBuildResultsDir(options) if buildResultsDir is None: return 1 print_verbose(f'Build result directory: {buildResultsDir}') # Find the archive with max debug build maxDebugArchive = findBuildArchive(buildResultsDir, 'MaxDebug') if maxDebugArchive is None: return 1 # Unpack the binary archive cmdLine = f'cd {buildResultsDir} && tar -xzf {maxDebugArchive}' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Unpacking the build archive return code: {ret}') if ret != 0: print_error(f'Unpacking the build archive {maxDebugArchive} error') return ret # Find the directory where the build result are unpacked maxDebugBuildResultDir = findBuildResultDir(buildResultsDir, 'MaxDebug') if maxDebugBuildResultDir is None: return 1 # Copy the binary cmdLine = f'cp -f {maxDebugBuildResultDir}bin/pubseq_gateway {artefactDir}' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Copy binary return code: {ret}') if ret != 0: print_error(f'Copy binary {maxDebugBuildResultDir}bin/pubseq_gateway error') return ret testDir = f'{buildResultsDir}c++/src/app/pubseq_gateway/server/test/' # Copy LMDB, ini, https files if prepareSandboxFiles(buildResultsDir, testDir, artefactDir) != 0: return 1 # Round up 1: Prepare the config file for UT (http) prepareConfigFile(artefactDir + 'pubseq_gateway.ini.baseline', artefactDir + 'pubseq_gateway.ini', {'si2csi.db': artefactDir + 'si2csi.db', 'bioseq_info.db': artefactDir +'bioseq_info.db', 'blob_prop.db': artefactDir + 'blob_prop.db', 'port': f'{PORT1}', 'ssl_enable': 'false', 'ssl_cert': '', 'ssl_key': ''}) # Run the server logFile = f'{artefactDir}pubseq_gateway-memcheck-ut-http.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine, stdout=subprocess.PIPE, stderr=subprocess.PIPE) time.sleep(AFTER_START_DELAY) # Run the tests errors = [] cmdLine = f'{testDir}run_tests.py localhost:{PORT1} > {artefactDir}memcheck-ut-http.log 2>&1' print_verbose(f'Running http unit tests: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'http unit tests return code: {ret}') if ret != 0: errors.append(f'Error executing http unit tests. Return code: {ret}') print_verbose('http unit tests failed') else: print_verbose('http unit tests succeeded') stopServer(SRV1, False) saveProcessArtefacts(SRV1, f'{artefactDir}memcheck-ut-http.sanitizer.log') SRV1 = None # Round up 2: Prepare the config file for UT (https) prepareConfigFile(artefactDir + 'pubseq_gateway.ini.baseline', artefactDir + 'pubseq_gateway.ini', {'si2csi.db': artefactDir + 'si2csi.db', 'bioseq_info.db': artefactDir +'bioseq_info.db', 'blob_prop.db': artefactDir + 'blob_prop.db', 'port': '10999', 'ssl_enable': 'true', 'ssl_cert': artefactDir + 'psg.crt', 'ssl_key': artefactDir + 'psg.key'}) # Run the server logFile = f'{artefactDir}pubseq_gateway-memcheck-ut-https.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine, stdout=subprocess.PIPE, stderr=subprocess.PIPE) time.sleep(AFTER_START_DELAY) # Run the tests cmdLine = f'{testDir}run_tests.py --https localhost:10999 > {artefactDir}memcheck-ut-https.log 2>&1' print_verbose(f'Running https unit tests: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'https unit tests return code: {ret}') if ret != 0: errors.append(f'Error executing https unit tests. Return code: {ret}') print_verbose('https unit tests failed') else: print_verbose('https unit tests succeeded') stopServer(SRV1, False) saveProcessArtefacts(SRV1, f'{artefactDir}memcheck-ut-https.sanitizer.log') SRV1 = None # Round up 3: Prepare the config file for stress test (http) prepareConfigFile(artefactDir + 'pubseq_gateway.ini.baseline', artefactDir + 'pubseq_gateway.ini', {'si2csi.db': artefactDir + 'si2csi.db', 'bioseq_info.db': artefactDir +'bioseq_info.db', 'blob_prop.db': artefactDir + 'blob_prop.db', 'port': f'{PORT1}', 'ssl_enable': 'false', 'ssl_cert': '', 'ssl_key': ''}) # Run the server logFile = f'{artefactDir}pubseq_gateway-memcheck-stress-http.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine, stdout=subprocess.PIPE, stderr=subprocess.PIPE) time.sleep(AFTER_START_DELAY) # Run the tests cmdLine = f'/usr/bin/bash {testDir}stresstests.sh --server localhost:{PORT1} > {artefactDir}memcheck-stress-http.log 2>&1' print_verbose(f'Running http stress tests: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'http stress tests return code: {ret}') if ret != 0: errors.append(f'Error executing http stress tests. Return code: {ret}') print_verbose('http stress tests failed') else: print_verbose('http stress tests succeeded') stopServer(SRV1, False) saveProcessArtefacts(SRV1, f'{artefactDir}memcheck-stress-http.sanitizer.log') SRV1 = None # Round up 4: Prepare the config file for stress tests (https) prepareConfigFile(artefactDir + 'pubseq_gateway.ini.baseline', artefactDir + 'pubseq_gateway.ini', {'si2csi.db': artefactDir + 'si2csi.db', 'bioseq_info.db': artefactDir +'bioseq_info.db', 'blob_prop.db': artefactDir + 'blob_prop.db', 'port': '10999', 'ssl_enable': 'true', 'ssl_cert': artefactDir + 'psg.crt', 'ssl_key': artefactDir + 'psg.key'}) # Run the server logFile = f'{artefactDir}pubseq_gateway-memcheck-stress-https.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine, stdout=subprocess.PIPE, stderr=subprocess.PIPE) time.sleep(AFTER_START_DELAY) # Run the tests cmdLine = f'/usr/bin/bash {testDir}stresstests.sh --https --server localhost:{PORT1} > {artefactDir}memcheck-stress-https.log 2>&1' print_verbose(f'Running https stress tests: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'https stress tests return code: {ret}') if ret != 0: errors.append(f'Error executing https stress tests. Return code: {ret}') print_verbose('https stress tests failed') else: print_verbose('https stress tests succeeded') stopServer(SRV1, False) saveProcessArtefacts(SRV1, f'{artefactDir}memcheck-stress-https.sanitizer.log') SRV1 = None # Round up 5: integration smoke test using psg_client prepareConfigFile(artefactDir + 'pubseq_gateway.ini.baseline', artefactDir + 'pubseq_gateway.ini', {'si2csi.db': artefactDir + 'si2csi.db', 'bioseq_info.db': artefactDir +'bioseq_info.db', 'blob_prop.db': artefactDir + 'blob_prop.db', 'port': '10999', 'ssl_enable': 'false', 'ssl_cert': '', 'ssl_key': ''}) psg_client_path = getPsgClientPath() if psg_client_path is None: errors.append('Cannot find psg_client. The integration smoke tests are skipped') else: cmdLine = f'cp -f {psg_client_path} {artefactDir}' print_verbose(f'Copying psg_client from {psg_client_path} ...') ret = os.system(cmdLine) if ret != 0: errors.append(f'Error copying psg_client from {psg_client_path}') else: # Run the server logFile = f'{artefactDir}pubseq_gateway-memcheck-integration-test.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine, stdout=subprocess.PIPE, stderr=subprocess.PIPE) time.sleep(AFTER_START_DELAY) # Run the tests integrationTestDir = f'{buildResultsDir}c++/src/app/pubseq_gateway/server/integrationsmoketest/' cmdLine = f'{integrationTestDir}integration_test -psgclient {artefactDir}psg_client localhost:{10999} > {artefactDir}memcheck-integration-test.log 2>&1' print_verbose(f'Running integration tests: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'integration tests return code: {ret}') if ret != 0: errors.append(f'Error executing integration tests. Return code: {ret}') print_verbose('integration tests failed') else: print_verbose('integration tests succeeded') stopServer(SRV1, False) saveProcessArtefacts(SRV1, f'{artefactDir}memcheck-integration-test.sanitizer.log') SRV1 = None if errors: print_error('Unit test errors:') for err in errors: print_error(err) return 1 return 0 def stress_test(commands, options, args): global SRV1 if prepareO2Gsandbox(options) != 0: return 1 artefactDir = getArtefactDir(options) # Round up 1: Prepare the config file for stress test (http) prepareConfigFile(artefactDir + 'pubseq_gateway.ini.baseline', artefactDir + 'pubseq_gateway.ini', {'si2csi.db': artefactDir + 'si2csi.db', 'bioseq_info.db': artefactDir +'bioseq_info.db', 'blob_prop.db': artefactDir + 'blob_prop.db', 'port': f'{PORT1}', 'ssl_enable': 'false', 'ssl_cert': '', 'ssl_key': ''}) # Run the server logFile = f'{artefactDir}pubseq_gateway-stress-http.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine) time.sleep(AFTER_START_DELAY) # Run the tests errors = [] buildResultsDir = getBuildResultsDir(options) testDir = f'{buildResultsDir}c++/src/app/pubseq_gateway/server/test/' cmdLine = f'/usr/bin/bash {testDir}stresstests.sh --server localhost:{PORT1} > {artefactDir}stress-http.log 2>&1' print_verbose(f'Running http stress tests: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'http stress tests return code: {ret}') if ret != 0: errors.append(f'Error executing http stress tests. Return code: {ret}') print_verbose('http stress tests failed') else: print_verbose('http stress tests succeeded') stopServer(SRV1) # Round up 2: Prepare the config file for UT (https) prepareConfigFile(artefactDir + 'pubseq_gateway.ini.baseline', artefactDir + 'pubseq_gateway.ini', {'si2csi.db': artefactDir + 'si2csi.db', 'bioseq_info.db': artefactDir +'bioseq_info.db', 'blob_prop.db': artefactDir + 'blob_prop.db', 'port': '10999', 'ssl_enable': 'true', 'ssl_cert': artefactDir + 'psg.crt', 'ssl_key': artefactDir + 'psg.key'}) # Run the server logFile = f'{artefactDir}pubseq_gateway-stress-https.log' cmdLine = [artefactDir + 'pubseq_gateway', '-conffile', artefactDir + 'pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine) time.sleep(AFTER_START_DELAY) # Run the tests cmdLine = f'/usr/bin/bash {testDir}stresstests.sh --https --server localhost:10999 > {artefactDir}stress-https.log 2>&1' print_verbose(f'Running https stress tests: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'https stress tests return code: {ret}') if ret != 0: errors.append(f'Error executing https stress tests. Return code: {ret}') print_verbose('https stress tests failed') else: print_verbose('https stress tests succeeded') stopServer(SRV1) if errors: print_error('Stress test errors:') for err in errors: print_error(err) return 1 return 0 def compare(commands, options, args): global SRV1 global SRV2 if prepareO2Gsandbox(options) != 0: return 1 artefactDir = getArtefactDir(options) # Prepare the server pointed by the --against-binary option oldVersion = options.against_binary if not oldVersion: print_error('Another PSG binary version must be provided ' 'via the --against-binary option') return 1 archivedBinaryPath = getArchivedBinaryPath(oldVersion) if archivedBinaryPath is None: return 1 cmdLine = f'cp -f {archivedBinaryPath} {artefactDir}pubseq_gateway-{oldVersion}-o2g' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Copy {archivedBinaryPath} file return code: {ret}') if ret != 0: print_error(f'Error copying {archivedBinaryPath} to {artefactDir}') return 1 h2loadPath = shutil.which('h2load') cmdLine = f'cp -f {h2loadPath} {artefactDir}h2load' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Copy {h2loadPath} file return code: {ret}') if ret != 0: print_error(f'Error copying {h2loadPath} to {artefactDir}') return 1 buildResultsDir = getBuildResultsDir(options) testDir = f'{buildResultsDir}c++/src/app/pubseq_gateway/server/test/' toCopy = ['performance.sh', 'psg_perf.py', 'aggregate.py'] for name in toCopy: cmdLine = f'cp -f {testDir}{name} {artefactDir}{name}' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Copy {name} file return code: {ret}') if ret != 0: print_error(f'Error copying {testDir}{name} to {artefactDir}') return 1 prepareConfigFile(f'{artefactDir}pubseq_gateway.ini.baseline', f'{artefactDir}pubseq_gateway.ini-{oldVersion}', {'si2csi.db': f'{artefactDir}si2csi.db', 'bioseq_info.db': f'{artefactDir}bioseq_info.db', 'blob_prop.db': f'{artefactDir}blob_prop.db', 'port': f'{PORT1}', 'ssl_enable': 'false', 'ssl_cert': '', 'ssl_key': ''}) # Run the old server logFile = f'{artefactDir}pubseq_gateway-compare-{oldVersion}.log' cmdLine = [f'{artefactDir}pubseq_gateway-{oldVersion}-o2g', '-conffile', f'{artefactDir}pubseq_gateway.ini-{oldVersion}', '-logfile', logFile] safeRemove(logFile) print_verbose('Running old PSG server: ' + ' '.join(cmdLine)) SRV1 = subprocess.Popen(cmdLine) time.sleep(AFTER_START_DELAY) # Prepare the currently built server prepareConfigFile(f'{artefactDir}pubseq_gateway.ini.baseline', f'{artefactDir}pubseq_gateway.ini', {'si2csi.db': f'{artefactDir}si2csi.db', 'bioseq_info.db': f'{artefactDir}bioseq_info.db', 'blob_prop.db': f'{artefactDir}blob_prop.db', 'port': f'{PORT2}', 'ssl_enable': 'false', 'ssl_cert': '', 'ssl_key': ''}) # Run the currently built server logFile = f'{artefactDir}pubseq_gateway-compare.log' cmdLine = [f'{artefactDir}pubseq_gateway', '-conffile', f'{artefactDir}pubseq_gateway.ini', '-logfile', logFile] safeRemove(logFile) print_verbose('Running current PSG server: ' + ' '.join(cmdLine)) SRV2 = subprocess.Popen(cmdLine) time.sleep(AFTER_START_DELAY) # Run the comparison script cmdLine = f'cd {artefactDir} && ./psg_perf.py localhost:{PORT1} localhost:{PORT2} > perf-v{oldVersion}-vs-v{options.psg_ver}.txt' print_verbose(f'Running performance comparison: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Comparison script return code: {ret}') if ret != 0: print_error(f'Error running comparison script. Return code: {ret}') else: print_verbose('Performance comparison script succeeded') stopServer(SRV1) stopServer(SRV2) return ret def archive(commands, options, args): artefactDir = getArtefactDir(options) buildResultsDir = getBuildResultsDir(options) if buildResultsDir is None: return 1 perfTestResultsDir = getPerfTestResultsPath(options) if perfTestResultsDir is None: return 1 cmdLine = ['prepare_release', 'archive', buildResultsDir] print_verbose('Running release archiving: ' + ' '.join(cmdLine)) proc = subprocess.Popen(cmdLine, stdin=subprocess.PIPE) time.sleep(5) out, err = proc.communicate(b'all\n') ret = proc.returncode if ret != 0: resultFile = artefactDir + 'archive-release.log' saveProcessArtefacts(proc, resultFile) print_error(f'Archiving error. Return code {ret}; results are in {resultFile}') return ret print_verbose('Archiving binary and source succeeded. Archive the perf_test results.') # Checking of the archive directory must be done only after the release is # archived archivedTestsDir = getArchivedTestPath(options) if archivedTestsDir is None: return 1 cmdLine = f'cp -rf {perfTestResultsDir} {archivedTestsDir}' print_verbose(f'Running perf test: {cmdLine}') ret = os.system(cmdLine) if ret != 0: print_error(f'Error archiving the perf_test results. Return code: {ret}') else: print_verbose('perf_test results have been archived successfully') return ret def production_tests(commands, options, args): devInstance = getDevServiceInstanceByVersion(options.psg_ver) if devInstance is None: # Error message has been printed in getDevServiceInstanceByVersion() return 1 print_verbose('Dev instance to use: ' + devInstance) # Get the psg_client, psg_client.py, testcases.json import ncbi.grid.deployment_info.parser as parser psg_client_path = parser.get_psg_client_path() psg_client_py_path = os.path.dirname(parser.get_psg_client_path()) + os.path.sep + 'psg_client.py' testcases_json_path = os.path.dirname(parser.get_psg_client_path()) + os.path.sep + 'testcases.json' # Form a command line for testcases.json cmdLine = f'{psg_client_py_path} testcases -binary {psg_client_path} ' \ f'-service {devInstance} -input-file {testcases_json_path} ' \ '-run on-ci -data-limit 1KB -preview-size 16 -timeout 60 -report' # Run print_verbose(f'Running production tests: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Production tests return code: {ret}') if ret != 0: print_error(f'Error running production tests. Return code: {ret}') else: print_verbose('Production tests succeeded') return ret def perf_test(commands, options, args): devInstance, prodInstance = getDevProdServerPair() if devInstance is None or prodInstance is None: print_error('Cannot find dev and prod PSG instances which are running on the same host') return 1 print_verbose('Dev instance to use: ' + devInstance) print_verbose('Prod instance to use: ' + prodInstance) artefactDir = getArtefactDir(options) buildResultsDir = getBuildResultsDir(options) testDir = f'{buildResultsDir}c++/src/app/pubseq_gateway/server/test/' toCopy = ['perf_test.sh'] for name in toCopy: cmdLine = f'cp -f {testDir}{name} {artefactDir}{name}' print_verbose(f'Command line to invoke: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Copy {name} file return code: {ret}') if ret != 0: print_error(f'Error copying {testDir}{name} to {artefactDir}') return 1 # Run the comparison script cmdLine = f'cd {artefactDir} && ./perf_test.sh {prodInstance} {devInstance} > perf_test.sh.log 2>&1' print_verbose(f'Running perf test: {cmdLine}') ret = os.system(cmdLine) print_verbose(f'Perf test script return code: {ret}') if ret != 0: print_error(f'Error running perf test script. Return code: {ret}') else: print_verbose('Perf test script succeeded') return ret if __name__ == '__main__': try: retVal = main() except KeyboardInterrupt: retVal = 2 print('Interrupted from the keyboard', file=sys.stderr) except Exception as exc: print(str(exc), file=sys.stderr) retVal = 3 if SRV1 is not None: stopServer(SRV1) if SRV2 is not None: stopServer(SRV2) sys.exit(retVal)