diff --git a/Framework/Built_In_Automation/Sequential_Actions/sequential_actions.py b/Framework/Built_In_Automation/Sequential_Actions/sequential_actions.py index dcb56e2a..abf6d245 100755 --- a/Framework/Built_In_Automation/Sequential_Actions/sequential_actions.py +++ b/Framework/Built_In_Automation/Sequential_Actions/sequential_actions.py @@ -143,6 +143,78 @@ def run(self, func, args, timeout): return payload +def _force_kill_hung_browser_sessions(): + """Best-effort kill of any live Selenium chromedriver/browser process trees. + + Called after an action_timeout abort, since Tear_Down_Selenium is never + reached for the timed-out action and driver.quit() itself can hang on the + same unresponsive chromedriver. Must never raise or block -- this runs + from inside a TimeoutError handler. + """ + try: + import psutil + except Exception: + return + + drivers = [] + try: + from Framework.Built_In_Automation.Web import utils as web_utils + for session in web_utils.get_browser_sessions().values(): + if isinstance(session, dict) and session.get("selenium_driver") is not None: + drivers.append(session["selenium_driver"]) + except Exception: + pass + + try: + from Framework.Built_In_Automation.Web.Selenium import BuiltInFunctions + for detail in BuiltInFunctions.selenium_details.values(): + if isinstance(detail, dict) and detail.get("driver") is not None: + drivers.append(detail["driver"]) + except Exception: + pass + + seen_pids = set() + for driver in drivers: + try: + pid = driver.service.process.pid + except Exception: + continue + if pid is None or pid in seen_pids: + continue + seen_pids.add(pid) + _kill_process_tree(pid) + + +def _kill_process_tree(pid): + try: + import psutil + root = psutil.Process(pid) + except Exception: + return + + try: + procs = root.children(recursive=True) + [root] + except Exception: + procs = [root] + + for proc in procs: + try: + proc.terminate() + except Exception: + pass + + try: + _gone, alive = psutil.wait_procs(procs, timeout=5) + except Exception: + alive = procs + + for proc in alive: + try: + proc.kill() + except Exception: + pass + + def _get_action_timeout(): """Resolve the `action_timeout` shared variable to seconds (float). @@ -193,6 +265,7 @@ async def _run_action_with_timeout(run_function, data_set): # thread won't block shutdown) and create a fresh worker for the next # action so its queues start clean. _action_worker = None + _force_kill_hung_browser_sessions() CommonUtil.ExecLog( sModuleInfo, "Action exceeded the action_timeout of %s second(s) and was aborted. " @@ -204,6 +277,7 @@ async def _run_action_with_timeout(run_function, data_set): try: return await asyncio.wait_for(result, timeout=timeout) except TimeoutError: + _force_kill_hung_browser_sessions() CommonUtil.ExecLog( sModuleInfo, "Action exceeded the action_timeout of %s second(s) and was aborted. " diff --git a/Framework/MainDriverApi.py b/Framework/MainDriverApi.py index f5daff51..4c0cf84f 100644 --- a/Framework/MainDriverApi.py +++ b/Framework/MainDriverApi.py @@ -722,9 +722,10 @@ async def run_all_test_steps_in_a_test_case( ) CommonUtil.CreateJsonReport(stepInfo=after_execution_dict) - with concurrent.futures.ThreadPoolExecutor() as executor: - thr = executor.submit(upload_step_report, run_id, test_case, this_step["step_sequence"], this_step["step_id"], after_execution_dict) - CommonUtil.SaveThread("step_report", thr) + executor = concurrent.futures.ThreadPoolExecutor() + thr = executor.submit(upload_step_report, run_id, test_case, this_step["step_sequence"], this_step["step_id"], after_execution_dict) + CommonUtil.SaveThread("step_report", thr) + executor.shutdown(wait=False) StepSeq += 1 CommonUtil.step_index += 1 diff --git a/Framework/Utilities/RequestFormatter.py b/Framework/Utilities/RequestFormatter.py index 2c979e1e..2b27af96 100644 --- a/Framework/Utilities/RequestFormatter.py +++ b/Framework/Utilities/RequestFormatter.py @@ -95,6 +95,7 @@ def renew_token(): r = session.post( url=form_uri("/zsvc/auth/v1/renew"), verify=False, + timeout=70, ) data = {} @@ -124,6 +125,7 @@ def login(): url=form_uri("/zsvc/auth/v1/login"), json=payload, verify=False, + timeout=70, )