Handle base dir in get manifest hash

clcache.py

@@ -97,8 +97,8 @@ def childDirectories(path, absolute=True):
 def normalizeBaseDir(baseDir):
     if baseDir:
         baseDir = os.path.normcase(baseDir)
-        if not baseDir.endswith(os.path.sep):
-            baseDir += os.path.sep
+        if baseDir.endswith(os.path.sep):
+            baseDir = baseDir[0:-1]
         return baseDir
     else:
         # Converts empty string to None
@@ -227,11 +227,25 @@ def getManifestHash(compilerBinary, commandLine, sourceFile):
         compilerHash = getCompilerHash(compilerBinary)
 
         # NOTE: We intentionally do not normalize command line to include
-        # preprocessor options. In direct mode we do not perform
-        # preprocessing before cache lookup, so all parameters are important.
-        # One of the few exceptions to this rule is the /MP switch, which only
-        # defines how many compiler processes are running simultaneusly.
+        # preprocessor options.  In direct mode we do not perform preprocessing
+        # before cache lookup, so all parameters are important.  One of the few
+        # exceptions to this rule is the /MP switch, which only defines how many
+        # compiler processes are running simultaneusly.  Arguments that specify
+        # the compiler where to find the source files are parsed to replace
+        # ocurrences of CLCACHE_BASEDIR by a placeholder.
         commandLine = [arg for arg in commandLine if not arg.startswith("/MP")]
+        arguments, inputFiles = CommandLineAnalyzer.parseArgumentsAndInputFiles(commandLine)
+        collapseBasedirInCmdPath = lambda path: collapseBasedirToPlaceholder(os.path.normcase(os.path.abspath(path)))
+
+        commandLine = []
+        argumentsWithPaths = ("AI", "I", "FU")
+        for k in sorted(arguments.keys()):
+            if k in argumentsWithPaths:
+                commandLine.extend(["/" + k + collapseBasedirInCmdPath(arg) for arg in arguments[k]])
+            else:
+                commandLine.extend(["/" + k + arg for arg in arguments[k]])
+
+        commandLine.extend(collapseBasedirInCmdPath(arg) for arg in inputFiles)
 
         additionalData = "{}|{}|{}".format(
             compilerHash, commandLine, ManifestRepository.MANIFEST_FILE_FORMAT_VERSION)

integrationtests.py

@@ -24,7 +24,7 @@
 
 PYTHON_BINARY = sys.executable
 CLCACHE_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clcache.py")
-ASSETS_DIR = os.path.join("tests", "integrationtests")
+ASSETS_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests", "integrationtests")
 
 # pytest-cov note: subprocesses are coverage tested by default with some limitations
 #   "For subprocess measurement environment variables must make it from the main process to the
@@ -1073,37 +1073,102 @@ def testHitsViaMpConcurrent(self):
 
 
 class TestBasedir(unittest.TestCase):
-    def testBasedir(self):
-        with cd(os.path.join(ASSETS_DIR, "basedir")), tempfile.TemporaryDirectory() as tempDir:
-            # First, create two separate build directories with the same sources
-            for buildDir in ["builddir_a", "builddir_b"]:
-                shutil.rmtree(buildDir, ignore_errors=True)
-                os.mkdir(buildDir)
+    def setUp(self):
+        self.projectDir = os.path.join(ASSETS_DIR, "basedir")
+        self.tempDir = tempfile.TemporaryDirectory()
+        self.clcacheDir = os.path.join(self.tempDir.name, "clcache")
+        self.savedCwd = os.getcwd()
+
+        os.chdir(self.tempDir.name)
+
+        # First, create two separate build directories with the same sources
+        for buildDir in ["builddir_a", "builddir_b"]:
+            shutil.copytree(self.projectDir, buildDir)
+
+        self.cache = clcache.Cache(self.clcacheDir)
+
+    def tearDown(self):
+        os.chdir(self.savedCwd)
+        self.tempDir.cleanup()
+
+    def _runCompiler(self, cppFile, extraArgs=None):
+        cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"]
+        if extraArgs:
+            cmd.extend(extraArgs)
+        cmd = cmd + [cppFile]
+        env = dict(os.environ, CLCACHE_DIR=self.clcacheDir, CLCACHE_BASEDIR=os.getcwd())
+        self.assertEqual(subprocess.call(cmd, env=env), 0)
+
+    def expectHit(self, runCompiler):
+        # Build once in one directory
+        with cd("builddir_a"):
+            runCompiler[0]()
+            with self.cache.statistics as stats:
+                self.assertEqual(stats.numCacheMisses(), 1)
+                self.assertEqual(stats.numCacheHits(), 0)
 
-                shutil.copy("main.cpp", buildDir)
-                shutil.copy("constants.h", buildDir)
+        # Build again in a different directory, this should hit now because of CLCACHE_BASEDIR
+        with cd("builddir_b"):
+            runCompiler[1]()
+            with self.cache.statistics as stats:
+                self.assertEqual(stats.numCacheMisses(), 1)
+                self.assertEqual(stats.numCacheHits(), 1)
 
-            cache = clcache.Cache(tempDir)
+    def expectMiss(self, runCompiler):
+        # Build once in one directory
+        with cd("builddir_a"):
+            runCompiler[0]()
+            with self.cache.statistics as stats:
+                self.assertEqual(stats.numCacheMisses(), 1)
+                self.assertEqual(stats.numCacheHits(), 0)
+
+        # Build again in a different directory, this should hit now because of CLCACHE_BASEDIR
+        with cd("builddir_b"):
+            runCompiler[1]()
+            with self.cache.statistics as stats:
+                self.assertEqual(stats.numCacheMisses(), 2)
+                self.assertEqual(stats.numCacheHits(), 0)
+
+    def testBasedirRelativePaths(self):
+        def runCompiler():
+            self._runCompiler("main.cpp")
+        self.expectHit([runCompiler, runCompiler])
+
+    def testBasedirAbsolutePaths(self):
+        def runCompiler():
+            self._runCompiler(os.path.join(os.getcwd(), "main.cpp"))
+        self.expectHit([runCompiler, runCompiler])
+
+    def testBasedirIncludeArg(self):
+        def runCompiler():
+            self._runCompiler("main.cpp", ["/I{}".format(os.getcwd())])
+        self.expectHit([runCompiler, runCompiler])
+
+    def testBasedirIncludeSlashes(self):
+        runCompiler1 = lambda: self._runCompiler("main.cpp", ["/I{}/".format(os.getcwd())])
+        runCompiler2 = lambda: self._runCompiler("main.cpp", ["/I{}".format(os.getcwd())])
+        self.expectHit([runCompiler1, runCompiler2])
+
+    def testBasedirIncludeArgDifferentCapitalization(self):
+        def runCompiler():
+            self._runCompiler("main.cpp", ["/I{}".format(os.getcwd().upper())])
+        self.expectHit([runCompiler, runCompiler])
+
+    def testBasedirDefineArg(self):
+        def runCompiler():
+            self._runCompiler("main.cpp", ["/DRESOURCES_DIR={}".format(os.getcwd())])
+        self.expectMiss([runCompiler, runCompiler])
+
+    def testBasedirRelativeIncludeArg(self):
+        basedir = os.getcwd()
+
+        def runCompiler(cppFile="main.cpp"):
+            cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c", "/I."]
+            cmd = cmd + [cppFile]
+            env = dict(os.environ, CLCACHE_DIR=self.clcacheDir, CLCACHE_BASEDIR=basedir)
+            self.assertEqual(subprocess.call(cmd, env=env), 0)
 
-            cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c", "main.cpp"]
-
-            # Build once in one directory
-            with cd("builddir_a"):
-                env = dict(os.environ, CLCACHE_DIR=tempDir, CLCACHE_BASEDIR=os.getcwd())
-                self.assertEqual(subprocess.call(cmd, env=env), 0)
-                with cache.statistics as stats:
-                    self.assertEqual(stats.numCacheMisses(), 1)
-                    self.assertEqual(stats.numCacheHits(), 0)
-
-            shutil.rmtree("builddir_a", ignore_errors=True)
-
-            # Build again in a different directory, this should hit now because of CLCACHE_BASEDIR
-            with cd("builddir_b"):
-                env = dict(os.environ, CLCACHE_DIR=tempDir, CLCACHE_BASEDIR=os.getcwd())
-                self.assertEqual(subprocess.call(cmd, env=env), 0)
-                with cache.statistics as stats:
-                    self.assertEqual(stats.numCacheMisses(), 1)
-                    self.assertEqual(stats.numCacheHits(), 1)
+        self.expectMiss([runCompiler, runCompiler])
 
 
 class TestCleanCache(unittest.TestCase):

unittests.py

@@ -69,14 +69,14 @@ def testNormalizeBaseDir(self):
         # Note: raw string literals cannot end in an odd number of backslashes
         # https://docs.python.org/3/faq/design.html#why-can-t-raw-strings-r-strings-end-with-a-backslash
         # So we consistenly use basic literals
-        self.assertEqual(clcache.normalizeBaseDir("c:"), "c:\\")
-        self.assertEqual(clcache.normalizeBaseDir("c:\\projects"), "c:\\projects\\")
+        self.assertEqual(clcache.normalizeBaseDir("c:"), "c:")
+        self.assertEqual(clcache.normalizeBaseDir("c:\\projects"), "c:\\projects")
 
-        self.assertEqual(clcache.normalizeBaseDir("C:\\"), "c:\\")
-        self.assertEqual(clcache.normalizeBaseDir("C:\\Projects\\"), "c:\\projects\\")
+        self.assertEqual(clcache.normalizeBaseDir("C:\\"), "c:")
+        self.assertEqual(clcache.normalizeBaseDir("C:\\Projects\\"), "c:\\projects")
 
-        self.assertEqual(clcache.normalizeBaseDir("c:\\projects with space"), "c:\\projects with space\\")
-        self.assertEqual(clcache.normalizeBaseDir("c:\\projects with ö"), "c:\\projects with ö\\")
+        self.assertEqual(clcache.normalizeBaseDir("c:\\projects with space"), "c:\\projects with space")
+        self.assertEqual(clcache.normalizeBaseDir("c:\\projects with ö"), "c:\\projects with ö")
 
     def testFilesBeneathSimple(self):
         with cd(os.path.join(ASSETS_DIR, "files-beneath")):