From 0e74206f1addd1322b953890e85dfed3f2bda41c Mon Sep 17 00:00:00 2001 From: Hubert Hesse Date: Thu, 11 Aug 2016 13:50:20 +0200 Subject: [PATCH 1/2] ManifestRepository unittest extract setUp method --- unittests.py | 45 +++++++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/unittests.py b/unittests.py index 3b5596a6..4beec3e0 100644 --- a/unittests.py +++ b/unittests.py @@ -175,14 +175,16 @@ def filesize(path, filename): return size + def setUp(self): + self.manifestsRootDir = os.path.join(ASSETS_DIR, "manifests") + self.mm = ManifestRepository(self.manifestsRootDir) + def testPaths(self): - manifestsRootDir = os.path.join(ASSETS_DIR, "manifests") - mm = ManifestRepository(manifestsRootDir) - ms = mm.section("fdde59862785f9f0ad6e661b9b5746b7") + ms = self.mm.section("fdde59862785f9f0ad6e661b9b5746b7") - self.assertEqual(ms.manifestSectionDir, os.path.join(manifestsRootDir, "fd")) + self.assertEqual(ms.manifestSectionDir, os.path.join(self.manifestsRootDir, "fd")) self.assertEqual(ms.manifestPath("fdde59862785f9f0ad6e661b9b5746b7"), - os.path.join(manifestsRootDir, "fd", "fdde59862785f9f0ad6e661b9b5746b7.json")) + os.path.join(self.manifestsRootDir, "fd", "fdde59862785f9f0ad6e661b9b5746b7.json")) def testIncludesContentHash(self): self.assertEqual( @@ -223,9 +225,6 @@ def testIncludesContentHash(self): ) def testStoreAndGetManifest(self): - manifestsRootDir = os.path.join(ASSETS_DIR, "manifests") - mm = ManifestRepository(manifestsRootDir) - manifest1 = Manifest([r'somepath\myinclude.h'], { "fdde59862785f9f0ad6e661b9b5746b7": "a649723940dc975ebd17167d29a532f8" }) @@ -233,8 +232,8 @@ def testStoreAndGetManifest(self): "474e7fc26a592d84dfa7416c10f036c6": "8771d7ebcf6c8bd57a3d6485f63e3a89" }) - ms1 = mm.section("8a33738d88be7edbacef48e262bbb5bc") - ms2 = mm.section("0623305942d216c165970948424ae7d1") + ms1 = self.mm.section("8a33738d88be7edbacef48e262bbb5bc") + ms2 = self.mm.section("0623305942d216c165970948424ae7d1") ms1.setManifest("8a33738d88be7edbacef48e262bbb5bc", manifest1) ms2.setManifest("0623305942d216c165970948424ae7d1", manifest2) @@ -250,16 +249,10 @@ def testStoreAndGetManifest(self): "8771d7ebcf6c8bd57a3d6485f63e3a89") def testNonExistingManifest(self): - manifestsRootDir = os.path.join(ASSETS_DIR, "manifests") - mm = ManifestRepository(manifestsRootDir) - - retrieved = mm.section("ffffffffffffffffffffffffffffffff").getManifest("ffffffffffffffffffffffffffffffff") + retrieved = self.mm.section("ffffffffffffffffffffffffffffffff").getManifest("ffffffffffffffffffffffffffffffff") self.assertIsNone(retrieved) def testClean(self): - manifestsRootDir = os.path.join(ASSETS_DIR, "manifests") - mm = ManifestRepository(manifestsRootDir) - # Size in (120, 240] bytes manifest1 = Manifest([r'somepath\myinclude.h'], { "fdde59862785f9f0ad6e661b9b5746b7": "a649723940dc975ebd17167d29a532f8" @@ -268,25 +261,25 @@ def testClean(self): manifest2 = Manifest([r'somepath\myinclude.h', 'moreincludes.h'], { "474e7fc26a592d84dfa7416c10f036c6": "8771d7ebcf6c8bd57a3d6485f63e3a89" }) - mm.section("8a33738d88be7edbacef48e262bbb5bc").setManifest("8a33738d88be7edbacef48e262bbb5bc", manifest1) - mm.section("0623305942d216c165970948424ae7d1").setManifest("0623305942d216c165970948424ae7d1", manifest2) + self.mm.section("8a33738d88be7edbacef48e262bbb5bc").setManifest("8a33738d88be7edbacef48e262bbb5bc", manifest1) + self.mm.section("0623305942d216c165970948424ae7d1").setManifest("0623305942d216c165970948424ae7d1", manifest2) - cleaningResultSize = mm.clean(240) + cleaningResultSize = self.mm.clean(240) # Only one of those manifests can be left self.assertLessEqual(cleaningResultSize, 240) - self.assertLessEqual(self._getDirectorySize(manifestsRootDir), 240) + self.assertLessEqual(self._getDirectorySize(self.manifestsRootDir), 240) - cleaningResultSize = mm.clean(240) + cleaningResultSize = self.mm.clean(240) # The one remaining is remains alive self.assertLessEqual(cleaningResultSize, 240) self.assertGreaterEqual(cleaningResultSize, 120) - self.assertLessEqual(self._getDirectorySize(manifestsRootDir), 240) - self.assertGreaterEqual(self._getDirectorySize(manifestsRootDir), 120) + self.assertLessEqual(self._getDirectorySize(self.manifestsRootDir), 240) + self.assertGreaterEqual(self._getDirectorySize(self.manifestsRootDir), 120) - cleaningResultSize = mm.clean(0) + cleaningResultSize = self.mm.clean(0) # All manifest are gone self.assertEqual(cleaningResultSize, 0) - self.assertEqual(self._getDirectorySize(manifestsRootDir), 0) + self.assertEqual(self._getDirectorySize(self.manifestsRootDir), 0) class TestCompilerArtifactsRepository(unittest.TestCase): From dd8f6881343a0455fc1fa132e3e14d72603c95a9 Mon Sep 17 00:00:00 2001 From: Hubert Hesse Date: Thu, 11 Aug 2016 13:50:44 +0200 Subject: [PATCH 2/2] Handle non existing includes in Manifest gracefully If clcache is called with a non existing include, force a cache miss, to see the compiler error --- clcache.py | 16 +++++++++++++--- unittests.py | 10 ++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/clcache.py b/clcache.py index cd299de8..4edc7dd1 100644 --- a/clcache.py +++ b/clcache.py @@ -186,6 +186,8 @@ def getManifestHash(compilerBinary, commandLine, sourceFile): @staticmethod def getIncludesContentHashForFiles(listOfIncludesAbsolute): listOfIncludesHashes = [getFileHash(filepath) for filepath in listOfIncludesAbsolute] + if None in listOfIncludesHashes: + return None return ManifestRepository.getIncludesContentHashForHashes(listOfIncludesHashes) @staticmethod @@ -425,6 +427,8 @@ def clean(self, stats, maximumSize): @staticmethod def getDirectCacheKey(manifestHash, includesContentHash): + if includesContentHash is None: + return None # We must take into account manifestHash to avoid # collisions when different source files use the same # set of includes. @@ -685,8 +689,11 @@ def getCompilerHash(compilerBinary): def getFileHash(filePath, additionalData=None): hasher = HashAlgorithm() - with open(filePath, 'rb') as inFile: - hasher.update(inFile.read()) + try: + with open(filePath, 'rb') as inFile: + hasher.update(inFile.read()) + except FileNotFoundError: + return None if additionalData is not None: # Encoding of this additional data does not really matter # as long as we keep it fixed, otherwise hashes change. @@ -1474,7 +1481,10 @@ def processDirect(cache, objectFile, compiler, cmdLine, sourceFile): # NOTE: command line options already included in hash for manifest name includesContentHash = ManifestRepository.getIncludesContentHashForFiles( [expandBasedirPlaceholder(include, baseDir) for include in manifest.includeFiles]) - cachekey = manifest.includesContentToObjectMap.get(includesContentHash) + if includesContentHash is None: + cachekey = None + else: + cachekey = manifest.includesContentToObjectMap.get(includesContentHash) if cachekey is not None: if cache.compilerArtifactsRepository.section(cachekey).hasEntry(cachekey): return processCacheHit(cache, objectFile, cachekey) diff --git a/unittests.py b/unittests.py index 4beec3e0..50e43d93 100644 --- a/unittests.py +++ b/unittests.py @@ -22,6 +22,7 @@ Manifest, ManifestRepository, Statistics, + getFileHash ) from clcache import ( AnalysisError, @@ -281,6 +282,15 @@ def testClean(self): self.assertEqual(cleaningResultSize, 0) self.assertEqual(self._getDirectorySize(self.manifestsRootDir), 0) + def testNonExistingInclude(self): + try: + self.mm.getIncludesContentHashForFiles(["include/removedInclude.h"]) + except Exception as e: + self.fail('Failed to handle non-existing includes: {0} {1} '.format(type(e).__name__,e)) + +class TestGlobalFunctions(unittest.TestCase): + def testgetFileHash(self): + self.assertEqual(getFileHash("nonexisting.h"), None) class TestCompilerArtifactsRepository(unittest.TestCase): def testPaths(self):