From 6c7c0a8fc09a8ea35ffd9ebfb663e2c479d1a1a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Si=C3=B1uela=20Pastor?= Date: Thu, 12 Oct 2017 12:49:53 -0700 Subject: [PATCH 1/3] Make flaky test run 100 times to help investigation --- integrationtests.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/integrationtests.py b/integrationtests.py index bcef1d45..eab2538a 100644 --- a/integrationtests.py +++ b/integrationtests.py @@ -807,18 +807,19 @@ def testHitsViaMpConcurrent(self): def testOutput(self): # type: () -> None - with cd(os.path.join(ASSETS_DIR, "parallel")), tempfile.TemporaryDirectory() as tempDir: - sources = glob.glob("*.cpp") - clcache.Cache(tempDir) - customEnv = self._createEnv(tempDir) - cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] - mpFlag = "/MP" + str(len(sources)) - out = subprocess.check_output(cmd + [mpFlag] + sources, env=customEnv).decode("ascii") - # print the output so that it shows up in py.test - print(out) - - for s in sources: - self.assertEqual(out.count(s), 1) + for _ in range(100): + with cd(os.path.join(ASSETS_DIR, "parallel")), tempfile.TemporaryDirectory() as tempDir: + sources = glob.glob("*.cpp") + clcache.Cache(tempDir) + customEnv = self._createEnv(tempDir) + cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] + mpFlag = "/MP" + str(len(sources)) + out = subprocess.check_output(cmd + [mpFlag] + sources, env=customEnv).decode("ascii") + # print the output so that it shows up in py.test + print(out) + + for s in sources: + self.assertEqual(out.count(s), 1) class TestRunParallel(RunParallelBase, unittest.TestCase): env = dict(os.environ) From ea3b7f0fb5ec340163e358f1adfa8574253ac8ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Si=C3=B1uela?= Date: Fri, 13 Oct 2017 17:08:24 +0200 Subject: [PATCH 2/3] Test work-around python 3.3 file handle inheritance --- clcache.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/clcache.py b/clcache.py index 860c28dd..4a6794a2 100644 --- a/clcache.py +++ b/clcache.py @@ -17,6 +17,7 @@ import hashlib import json import multiprocessing +import msvcrt import os import pickle import re @@ -31,6 +32,7 @@ HashAlgorithm = hashlib.md5 OUTPUT_LOCK = threading.Lock() +PROCESS_LOCK = threading.Lock() # try to use os.scandir or scandir.scandir # fall back to os.listdir if not found @@ -67,6 +69,7 @@ # Define some Win32 API constants here to avoid dependency on win32pipe NMPWAIT_WAIT_FOREVER = wintypes.DWORD(0xFFFFFFFF) ERROR_PIPE_BUSY = 231 +HANDLE_FLAG_INHERIT = 1 # ManifestEntry: an entry in a manifest file # `includeFiles`: list of paths to include files, which this source file uses @@ -1324,13 +1327,26 @@ def invokeRealCompiler(compilerBinary, cmdLine, captureOutput=False, outputAsStr if captureOutput: # Don't use subprocess.communicate() here, it's slow due to internal # threading. - with TemporaryFile() as stdoutFile, TemporaryFile() as stderrFile: + with PROCESS_LOCK: + stdoutFile = TemporaryFile() + stderrFile = TemporaryFile() + + # Make temporary file handles non-inheritable, this is a workaround to + # a problem in Python 3.3 + windll.kernel32.SetHandleInformation(msvcrt.get_osfhandle(stdoutFile.fileno()), HANDLE_FLAG_INHERIT, 0) + windll.kernel32.SetHandleInformation(msvcrt.get_osfhandle(stderrFile.fileno()), HANDLE_FLAG_INHERIT, 0) + compilerProcess = subprocess.Popen(realCmdline, stdout=stdoutFile, stderr=stderrFile, env=environment) - returnCode = compilerProcess.wait() - stdoutFile.seek(0) - stdout = stdoutFile.read() - stderrFile.seek(0) - stderr = stderrFile.read() + + try: + returnCode = compilerProcess.wait() + stdoutFile.seek(0) + stdout = stdoutFile.read() + stderrFile.seek(0) + stderr = stderrFile.read() + finally: + stdoutFile.close() + stderrFile.close() else: returnCode = subprocess.call(realCmdline, env=environment) From c760f8ca5d098126472ea5b155bc2f955884a190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Si=C3=B1uela=20Pastor?= Date: Mon, 16 Oct 2017 09:28:29 +0200 Subject: [PATCH 3/3] Fix indentation This code should not run under PROCESS_LOCK. --- clcache.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/clcache.py b/clcache.py index 4a6794a2..5fcc8219 100644 --- a/clcache.py +++ b/clcache.py @@ -1338,15 +1338,15 @@ def invokeRealCompiler(compilerBinary, cmdLine, captureOutput=False, outputAsStr compilerProcess = subprocess.Popen(realCmdline, stdout=stdoutFile, stderr=stderrFile, env=environment) - try: - returnCode = compilerProcess.wait() - stdoutFile.seek(0) - stdout = stdoutFile.read() - stderrFile.seek(0) - stderr = stderrFile.read() - finally: - stdoutFile.close() - stderrFile.close() + try: + returnCode = compilerProcess.wait() + stdoutFile.seek(0) + stdout = stdoutFile.read() + stderrFile.seek(0) + stderr = stderrFile.read() + finally: + stdoutFile.close() + stderrFile.close() else: returnCode = subprocess.call(realCmdline, env=environment)