diff --git a/clcache.py b/clcache.py index 530fdb04..ff9ee062 100644 --- a/clcache.py +++ b/clcache.py @@ -995,6 +995,7 @@ def findCompilerBinary(): def printTraceStatement(msg): + # type: (str) -> None if "CLCACHE_LOG" in os.environ: scriptDir = os.path.realpath(os.path.dirname(sys.argv[0])) with OUTPUT_LOCK: @@ -1243,15 +1244,21 @@ def parseArgumentsAndInputFiles(cmdline): @staticmethod def analyze(cmdline): + # type: List[str] -> Tuple[List[Tuple[str, str]], List[str]] options, inputFiles = CommandLineAnalyzer.parseArgumentsAndInputFiles(cmdline) + # Use an override pattern to shadow input files that have + # already been specified in the function above + inputFiles = {inputFile: '' for inputFile in inputFiles} compl = False if 'Tp' in options: - inputFiles += options['Tp'] + inputFiles.update({inputFile: '/Tp' for inputFile in options['Tp']}) compl = True if 'Tc' in options: - inputFiles += options['Tc'] + inputFiles.update({inputFile: '/Tc' for inputFile in options['Tc']}) compl = True + # Now collect the inputFiles into the return format + inputFiles = list(inputFiles.items()) if not inputFiles: raise NoSourceFileError() @@ -1284,7 +1291,7 @@ def analyze(cmdline): objectFiles = [tmp] if objectFiles is None: # Generate from .c/.cpp filenames - objectFiles = [os.path.join(prefix, basenameWithoutExtension(f)) + '.obj' for f in inputFiles] + objectFiles = [os.path.join(prefix, basenameWithoutExtension(f)) + '.obj' for f, _ in inputFiles] printTraceStatement("Compiler source files: {}".format(inputFiles)) printTraceStatement("Compiler object file: {}".format(objectFiles)) @@ -1612,19 +1619,26 @@ def processCompileRequest(cache, compiler, args): printOutAndErr(out, err) return exitCode +def filterSourceFiles(cmdLine, sourceFiles): + # type: (List[str], List[Tuple[str, str]]) -> Iterator[str] + setOfSources = set(sourceFile for sourceFile, _ in sourceFiles) + skippedArgs = ('/Tc', '/Tp', '-Tp', '-Tc') + yield from ( + arg for arg in cmdLine + if not (arg in setOfSources or arg.startswith(skippedArgs)) + ) + def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles): - baseCmdLine = [] - setOfSources = set(sourceFiles) - for arg in cmdLine: - if not (arg in setOfSources or arg.startswith("/MP")): - baseCmdLine.append(arg) + # type: (Any, str, List[str], Any, List[Tuple[str, str]], List[str]) -> int + # Filter out all source files from the command line to form baseCmdLine + baseCmdLine = [arg for arg in filterSourceFiles(cmdLine, sourceFiles) if not arg.startswith('/MP')] exitCode = 0 cleanupRequired = False with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor: jobs = [] - for srcFile, objFile in zip(sourceFiles, objectFiles): - jobCmdLine = baseCmdLine + [srcFile] + for (srcFile, srcLanguage), objFile in zip(sourceFiles, objectFiles): + jobCmdLine = baseCmdLine + [srcLanguage + srcFile] jobs.append(executor.submit( processSingleSource, compiler, jobCmdLine, srcFile, objFile, environment)) diff --git a/integrationtests.py b/integrationtests.py index 33400443..bcef1d45 100644 --- a/integrationtests.py +++ b/integrationtests.py @@ -806,6 +806,7 @@ def testHitsViaMpConcurrent(self): self.assertEqual(stats.numCacheEntries(), 2) def testOutput(self): + # type: () -> None with cd(os.path.join(ASSETS_DIR, "parallel")), tempfile.TemporaryDirectory() as tempDir: sources = glob.glob("*.cpp") clcache.Cache(tempDir) @@ -813,6 +814,8 @@ def testOutput(self): cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] mpFlag = "/MP" + str(len(sources)) out = subprocess.check_output(cmd + [mpFlag] + sources, env=customEnv).decode("ascii") + # print the output so that it shows up in py.test + print(out) for s in sources: self.assertEqual(out.count(s), 1) diff --git a/unittests.py b/unittests.py index 05190df1..131ee60f 100644 --- a/unittests.py +++ b/unittests.py @@ -488,6 +488,30 @@ def testNestedResponseFiles(self): ) +class TestFilterSourceFiles(unittest.TestCase): + def _assertFiltered(self, cmdLine, files, filteredCmdLine): + # type: (List[str], List[Tuple[str, str]]) -> List[str] + files = clcache.filterSourceFiles(cmdLine, files) + self.assertEqual(list(files), filteredCmdLine) + + def testFilterSourceFiles(self): + self._assertFiltered( + ['/c', '/EP', '/FoSome.obj', 'main.cpp'], [('main.cpp', '')], + ['/c', '/EP', '/FoSome.obj']) + self._assertFiltered( + ['/c', '/EP', '/FoSome.obj', 'main.cpp'], [('main.cpp', '/Tp')], + ['/c', '/EP', '/FoSome.obj']) + self._assertFiltered( + ['/c', '/EP', '/FoSome.obj', 'main.cpp'], [('main.cpp', '/Tc')], + ['/c', '/EP', '/FoSome.obj']) + self._assertFiltered( + ['/c', '/EP', '/FoSome.obj', '/Tcmain.cpp'], [('main.cpp', '')], + ['/c', '/EP', '/FoSome.obj']) + self._assertFiltered( + ['/c', '/EP', '/FoSome.obj', '/Tcmain.cpp'], [('main.cpp', '-Tc')], + ['/c', '/EP', '/FoSome.obj']) + + class TestAnalyzeCommandLine(unittest.TestCase): def _testSourceFilesOk(self, cmdLine): try: @@ -504,13 +528,14 @@ def _testFailure(self, cmdLine, expectedExceptionClass): self.assertRaises(expectedExceptionClass, lambda: CommandLineAnalyzer.analyze(cmdLine)) def _testFull(self, cmdLine, expectedSourceFiles, expectedOutputFile): + # type: (List[str], List[Tuple[str, str]], List[str]) -> None sourceFiles, outputFile = CommandLineAnalyzer.analyze(cmdLine) self.assertEqual(sourceFiles, expectedSourceFiles) self.assertEqual(outputFile, expectedOutputFile) def _testFo(self, foArgument, expectedObjectFilepath): self._testFull(['/c', foArgument, 'main.cpp'], - ["main.cpp"], [expectedObjectFilepath]) + [("main.cpp", '')], [expectedObjectFilepath]) def _testFi(self, fiArgument): self._testPreprocessingOutfile(['/c', '/P', fiArgument, 'main.cpp']) @@ -528,7 +553,7 @@ def testEmpty(self): self._testFailure([], NoSourceFileError) def testSimple(self): - self._testFull(["/c", "main.cpp"], ["main.cpp"], ["main.obj"]) + self._testFull(["/c", "main.cpp"], [("main.cpp", "")], ["main.obj"]) def testNoSource(self): # No source file has priority over other errors, for consistency @@ -547,7 +572,7 @@ def testNoSource(self): def testOutputFileFromSourcefile(self): # For object file self._testFull(['/c', 'main.cpp'], - ['main.cpp'], ['main.obj']) + [('main.cpp', '')], ['main.obj']) # For preprocessor file self._testFailure(['/c', '/P', 'main.cpp'], CalledForPreprocessingError) @@ -634,9 +659,9 @@ def testPreprocessingFi(self): def testTpTcSimple(self): # clcache can handle /Tc or /Tp as long as there is only one of them self._testFull(['/c', '/TcMyCcProgram.c'], - ['MyCcProgram.c'], ['MyCcProgram.obj']) + [('MyCcProgram.c', '/Tc')], ['MyCcProgram.obj']) self._testFull(['/c', '/TpMyCxxProgram.cpp'], - ['MyCxxProgram.cpp'], ['MyCxxProgram.obj']) + [('MyCxxProgram.cpp', '/Tp')], ['MyCxxProgram.obj']) def testLink(self): self._testFailure(["main.cpp"], CalledForLinkError)