blob: d9c71afbf3589422936ad61ccc6818ca54a5459f [file] [log] [blame]
Jeff Gastona7df0922021-06-02 17:39:47 -04001#!/usr/bin/python3
2
3"""
4Parses information about failing tests, and then generates a change to disable them.
5
6Requires that the `bugged` command-line tool is installed, see go/bugged .
7"""
8
9import argparse, csv, os, subprocess
10
11parser = argparse.ArgumentParser(
12description=__doc__
13)
Jeff Gastona7c77722023-11-02 15:15:59 -040014parser.add_argument("-v", help="Verbose", action="store_true")
Jeff Gastona7df0922021-06-02 17:39:47 -040015
16dirOfThisScript = os.path.dirname(os.path.realpath(__file__))
17supportRoot = os.path.dirname(dirOfThisScript)
18
Jeff Gastona7c77722023-11-02 15:15:59 -040019logger = None
20
21class PrintLogger(object):
22def log(self, message):
23print(message)
24
25class DisabledLogger(object):
26def log(self, message):
27pass
28
29def log(message):
30logger.log(message)
31
Jeff Gastona7df0922021-06-02 17:39:47 -040032class LocatedFailure(object):
Jeff Gastona7c77722023-11-02 15:15:59 -040033def __init__(self, failure, location, bugId):
Jeff Gastona7df0922021-06-02 17:39:47 -040034self.failure = failure
35self.location = location
Jeff Gastona7c77722023-11-02 15:15:59 -040036self.bugId = bugId
Jeff Gastona7df0922021-06-02 17:39:47 -040037
38class TestFailure(object):
Jeff Gastona7c77722023-11-02 15:15:59 -040039def __init__(self, qualifiedClassName, methodName, testDefinitionName, branchName, testFailureUrl, bugId):
Jeff Gastona7df0922021-06-02 17:39:47 -040040self.qualifiedClassName = qualifiedClassName
41self.methodName = methodName
42self.testDefinitionName = testDefinitionName
Jeff Gastona7df0922021-06-02 17:39:47 -040043self.branchName = branchName
Jeff Gastona7c77722023-11-02 15:15:59 -040044self.failureUrl = testFailureUrl
45self.bugId = bugId
Jeff Gastona7df0922021-06-02 17:39:47 -040046
47def getUrl(self):
Jeff Gastona7c77722023-11-02 15:15:59 -040048return self.testFailureUrl
Jeff Gastona7df0922021-06-02 17:39:47 -040049
50class FailuresDatabase(object):
51"""A collection of LocatedFailure instances, organized by their locations"""
52def __init__(self):
53self.failuresByPath = {}
54
55def add(self, locatedFailure):
56path = locatedFailure.location.filePath
57if path not in self.failuresByPath:
58self.failuresByPath[path] = {}
59failuresAtPath = self.failuresByPath[path]
60
61lineNumber = locatedFailure.location.lineNumber
62if lineNumber not in failuresAtPath:
63failuresAtPath[lineNumber] = locatedFailure
Jeff Gastona7df0922021-06-02 17:39:47 -040064
65# returns Map<String, LocatedFailure> with key being filePath
66def getAll(self):
67results = {}
68for path, failuresAtPath in self.failuresByPath.items():
69lineNumbers = sorted(failuresAtPath.keys(), reverse=True)
70resultsAtPath = []
71# add failures in reverse order to make it easier to modify methods without adjusting line numbers for other methods
72for line in lineNumbers:
73resultsAtPath.append(failuresAtPath[line])
74results[path] = resultsAtPath
75return results
76
Jeff Gastona7c77722023-11-02 15:15:59 -040077def parseBugLine(bugId, line):
78components = line.split(" | ")
79if len(components) < 3:
80return None
81testLink = components[1]
82# Example test link: [compose-ui-uidebugAndroidTest.xml androidx.compose.ui.window.PopupAlignmentTest#popup_correctPosition_alignmentTopCenter_rtl](https://android-build.googleplex.com/builds/tests/view?testResultId=TR96929024659298098)
83closeBracketIndex = testLink.rindex("]")
84if closeBracketIndex <= 0:
85raise Exception("Failed to parse b/" + bugId + " '" + line + "', testLink '" + testLink + "', closeBracketIndex = " + str(closeBracketIndex))
86linkText = testLink[1:closeBracketIndex]
87linkDest = testLink[closeBracketIndex + 1:]
88# Example linkText: compose-ui-uidebugAndroidTest.xml androidx.compose.ui.window.PopupAlignmentTest#popup_correctPosition_alignmentTopCenter_rtl
89# Example linkDest: (https://android-build.googleplex.com/builds/tests/view?testResultId=TR96929024659298098)
90testResultUrl = linkDest.replace("(", "").replace(")", "")
91# Example testResultUrl: https://android-build.googleplex.com/builds/tests/view?testResultId=TR96929024659298098
92spaceIndex = linkText.index(" ")
93if spaceIndex <= 0:
94raise Exception("Failed to parse b/" + bugId + " '" + line + "', linkText = '" + linkText + ", spaceIndex = " + str(spaceIndex))
95testDefinitionName = linkText[:spaceIndex]
96testPath = linkText[spaceIndex+1:]
97# Example test path: androidx.compose.ui.window.PopupAlignmentTest#popup_correctPosition_alignmentTopCenter_rtl
98testPathSplit = testPath.split("#")
99if len(testPathSplit) != 2:
100raise Exception("Failed to parse b/" + bugId + " '" + line + "', testPath = '" + testPath + "', len(testPathSplit) = " + str(len(testPathSplit)))
101testClass, testMethod = testPathSplit
102
103branchName = components[2].strip()
104print(" parsed test failure class=" + testClass + " method='" + testMethod + "' definition=" + testDefinitionName + " branch=" + branchName + " failureUrl=" + testResultUrl + " bugId=" + bugId)
105return TestFailure(testClass, testMethod, testDefinitionName, branchName, testResultUrl, bugId)
106
107def parseBug(bugId):
108bugText = shellRunner.runAndGetOutput(["bugged", "show", bugId])
109log("bug text = '" + bugText + "'")
Jeff Gastona7df0922021-06-02 17:39:47 -0400110failures = []
Jeff Gastona7c77722023-11-02 15:15:59 -0400111bugLines = bugText.split("\n")
112
113stillFailing = True
114listingTests = False
115for i in range(len(bugLines)):
116line = bugLines[i]
117#log("parsing bug line " + line)
118if listingTests:
119failure = parseBugLine(bugId, line)
120if failure is not None:
121failures.append(failure)
122if "---|---|---|---|---" in line: # table start
123listingTests = True
124if " # " in line: # start of new section
125listingTests = False
126if "There are no more failing tests in this regression" in line or "ATN has not seen a failure for this regression recently." in line or "This regression has been identified as a duplicate of another one" in line:
127stillFailing = False
128if len(failures) < 1:
129raise Exception("Failed to parse b/" + bugId + ": identified 0 failures. Rerun with -v for more information")
130if not stillFailing:
131print("tests no longer failing")
132return []
133return failures
134
135# identifies failing tests
136def getFailureData():
137bugsQuery = ["bugged", "search", "hotlistid:5083126 status:open", "--columns", "issue"]
138print("Searching for bugs: " + str(bugsQuery))
139bugsOutput = shellRunner.runAndGetOutput(bugsQuery)
140bugIds = bugsOutput.split("\n")
141print("Checking " + str(len(bugIds)) + " bugs")
142failures = []
143for i in range(len(bugIds)):
144bugId = bugIds[i].strip()
145if bugId != "issue" and bugId != "":
146print("")
147print("Parsing bug " + bugId + " (" + str(i) + "/" + str(len(bugIds)) + ")")
148failures += parseBug(bugId)
Jeff Gastona7df0922021-06-02 17:39:47 -0400149return failures
150
151class FileLocation(object):
152def __init__(self, filePath, lineNumber):
153self.filePath = filePath
154self.lineNumber = lineNumber
155
156def __str__(self):
157return self.filePath + "#" + str(self.lineNumber)
158
159class ShellRunner(object):
160def __init__(self):
161return
162
163def runAndGetOutput(self, args):
164result = subprocess.run(args, capture_output=True, text=True).stdout
165return result
166
167def run(self, args):
168subprocess.run(args, capture_output=False)
169
170shellRunner = ShellRunner()
171
172class FileFinder(object):
173def __init__(self, rootPath):
174self.rootPath = rootPath
175self.resultsCache = {}
176
177def findIname(self, name):
178if name not in self.resultsCache:
179text = shellRunner.runAndGetOutput(["find", self.rootPath , "-type", "f", "-iname", name])
180filePaths = [path.strip() for path in text.split("\n")]
181filePaths = [path for path in filePaths if path != ""]
182self.resultsCache[name] = filePaths
183return self.resultsCache[name]
184fileFinder = FileFinder(supportRoot)
185
186class ClassFinder(object):
187"""Locates the file path and line number for classes and methods"""
188def __init__(self):
189self.classToFile_cache = {}
190self.methodLocations_cache = {}
191
192def findMethod(self, qualifiedClassName, methodName):
193bracketIndex = methodName.find("[")
194if bracketIndex >= 0:
195methodName = methodName[:bracketIndex]
196fullName = qualifiedClassName + "." + methodName
197containingFile = self.findFileContainingClass(qualifiedClassName)
198if containingFile is None:
199return None
200if fullName not in self.methodLocations_cache:
201index = -1
202foundLineNumber = None
203with open(containingFile) as f:
204for line in f:
205index += 1
206if (" " + methodName + "(") in line:
207if foundLineNumber is not None:
208# found two matches, can't choose one
209foundLineNumber = None
210break
211foundLineNumber = index
212result = None
213if foundLineNumber is not None:
214result = FileLocation(containingFile, foundLineNumber)
215self.methodLocations_cache[fullName] = result
216return self.methodLocations_cache[fullName]
217
218
219def findFileContainingClass(self, qualifiedName):
220if qualifiedName not in self.classToFile_cache:
221lastDotIndex = qualifiedName.rindex(".")
222if lastDotIndex >= 0:
223packageName = qualifiedName[:lastDotIndex]
224className = qualifiedName[lastDotIndex + 1:]
225else:
226packageName = ""
227className = qualifiedName
228options = fileFinder.findIname(className + ".*")
229possibleContainingFiles = sorted(options)
230result = None
231for f in possibleContainingFiles:
232if self.getPackage(f) == packageName:
233result = f
234break
235self.classToFile_cache[qualifiedName] = result
236return self.classToFile_cache[qualifiedName]
237
238def getPackage(self, filePath):
239prefix = "package "
240with open(filePath) as f:
241for line in f:
242line = line.strip()
243if line.startswith(prefix):
244suffix = line[len(prefix):]
245if suffix.endswith(";"):
246return suffix[:-1]
247return suffix
248return None
249
250classFinder = ClassFinder()
251
252def readFile(path):
253f = open(path)
254text = f.read()
255f.close()
256return text
257
258def writeFile(path, text):
259f = open(path, "w")
260f.write(text)
261f.close()
262
263def extractIndent(text):
264indentSize = 0
265for c in text:
266if c == " ":
267indentSize += 1
268else:
269break
270return " " * indentSize
271
272class SourceFile(object):
273"""An in-memory model of a source file (java, kotlin) that can be manipulated and saved"""
274def __init__(self, path):
275text = readFile(path)
276self.lines = text.split("\n")
277self.path = path
278
279def isKotlin(self):
280return self.path.endswith(".kt")
281
282def maybeSemicolon(self):
283if self.isKotlin():
284return ""
285return ";"
286
287def addAnnotation(self, methodLineNumber, annotation):
288parenIndex = annotation.find("(")
289if parenIndex > 0:
290baseName = annotation[:parenIndex]
291else:
292baseName = annotation
293if self.findAnnotationLine(methodLineNumber, baseName) is not None:
294# already have an annotation, don't need to add another
295return
296indent = extractIndent(self.lines[methodLineNumber])
297self.insertLine(methodLineNumber, indent + annotation)
298
299# Adds an import to this file
300# Attempts to preserve alphabetical import ordering:
301# If two consecutive imports can be found such that one should precede this import and
302# one should follow this import, inserts between those two imports
303# Otherwise attempts to add this import after the last import or before the first import
304# (Note that imports might be grouped into multiple blocks, each separated by a blank line)
305def addImport(self, symbolText):
306insertText = "import " + symbolText + self.maybeSemicolon()
307if insertText in self.lines:
308return # already added
309# set of lines that the insertion could immediately precede
310beforeLineNumbers = set()
311# set of lines that the insertion could immediately follow
312afterLineNumbers = set()
313for i in range(len(self.lines)):
314line = self.lines[i]
315if line.startswith("import"):
316# found an import. Should our import be before or after?
317if insertText < line:
318beforeLineNumbers.add(i)
319else:
320afterLineNumbers.add(i)
321# search for two adjacent lines that the line can be inserted between
322insertionLineNumber = None
323for i in range(len(self.lines) - 1):
324if i in afterLineNumbers and (i + 1) in beforeLineNumbers:
325insertionLineNumber = i + 1
326break
327# search for a line we can insert after
328if insertionLineNumber is None:
329for i in range(len(self.lines) - 1):
330if i in afterLineNumbers and (i + 1) not in afterLineNumbers:
331insertionLineNumber = i + 1
332break
333# search for a line we can insert before
334if insertionLineNumber is None:
335for i in range(len(self.lines) - 1, 0, -1):
336if i in beforeLineNumbers and (i - 1) not in beforeLineNumbers:
337insertionLineNumber = i
338break
339
340if insertionLineNumber is not None:
341self.insertLine(insertionLineNumber, insertText)
342
343def insertLine(self, beforeLineNumber, text):
344self.lines = self.lines[:beforeLineNumber] + [text] + self.lines[beforeLineNumber:]
345
346def findAnnotationLine(self, methodLineNumber, annotationText):
347lineNumber = methodLineNumber
348while True:
349if lineNumber < 0:
350return None
351if annotationText in self.lines[lineNumber]:
352return lineNumber
353if self.lines[lineNumber].strip() == "":
354return None
355lineNumber -= 1
356
357def removeLine(self, index):
358self.lines = self.lines[:index] + self.lines[index + 1:]
359
360def hasAnnotation(self, methodLineNumber, annotation):
361return self.findAnnotationLine(methodLineNumber, annotation) is not None
362
363def save(self):
364writeFile(self.path, "\n".join(self.lines))
365
Jeff Gastona7df0922021-06-02 17:39:47 -0400366# converts from a List<TestFailure> to a FailuresDatabase containing LocatedFailure
367def locate(failures):
368db = FailuresDatabase()
369for failure in failures:
370location = classFinder.findMethod(failure.qualifiedClassName, failure.methodName)
371if location is not None:
Jeff Gastona7c77722023-11-02 15:15:59 -0400372db.add(LocatedFailure(failure, location, failure.bugId))
Jeff Gastona7df0922021-06-02 17:39:47 -0400373else:
Jeff Gastona7c77722023-11-02 15:15:59 -0400374message = "Could not locate " + str(failure.qualifiedClassName) + "#" + str(failure.methodName) + " for " + str(failure.bugId)
Jeff Gastona7df0922021-06-02 17:39:47 -0400375if failure.branchName != "aosp-androidx-main":
376message += ", should be in " + failure.branchName
377print(message)
378return db
379
Jeff Gastona7df0922021-06-02 17:39:47 -0400380# Given a FailureDatabase, disables all of the tests mentioned in it, by adding the appropriate
381# annotations:
Jeff Gastona7c77722023-11-02 15:15:59 -0400382# failures get annotated with @Ignore ,
Jeff Gastona7df0922021-06-02 17:39:47 -0400383# Annotations link to the associated bug if possible
384def disable(failuresDatabase):
Jeff Gastona7df0922021-06-02 17:39:47 -0400385numUpdates = 0
386failuresByPath = failuresDatabase.getAll()
387for path, failuresAtPath in failuresByPath.items():
388source = SourceFile(path)
389addedIgnore = False
Jeff Gastona7df0922021-06-02 17:39:47 -0400390for failure in failuresAtPath:
391lineNumber = failure.location.lineNumber
Jeff Gastona7c77722023-11-02 15:15:59 -0400392if source.hasAnnotation(lineNumber, "@Ignore"):
Jeff Gastona7df0922021-06-02 17:39:47 -0400393continue
Jeff Gastona7c77722023-11-02 15:15:59 -0400394bugId = failure.bugId
395bugText = '"b/' + bugId + '"'
396source.addAnnotation(lineNumber, "@Ignore(" + bugText + ")")
397addedIgnore = True
Jeff Gastona7df0922021-06-02 17:39:47 -0400398if addedIgnore:
399source.addImport("org.junit.Ignore")
Jeff Gastona7df0922021-06-02 17:39:47 -0400400source.save()
401numUpdates += 1
Jeff Gastona7c77722023-11-02 15:15:59 -0400402print("Made " + str(numUpdates) + " updates")
403
404def commit():
405print("Generating git commit per OWNERS file")
406os.chdir(supportRoot)
407commitMessage = """Autogenerated suppression of test failures
Jeff Gastona7df0922021-06-02 17:39:47 -0400408
409This commit was created with the help of development/suppressFailingTests.py
Jeff Gastona7df0922021-06-02 17:39:47 -0400410"""
Jeff Gastona7c77722023-11-02 15:15:59 -0400411shellRunner.run(["development/split_change_into_owners.sh", commitMessage])
Jeff Gastona7df0922021-06-02 17:39:47 -0400412
Jeff Gastona7df0922021-06-02 17:39:47 -0400413
414def main():
Jeff Gastona7c77722023-11-02 15:15:59 -0400415global logger
416arguments = parser.parse_args()
417if arguments.v:
418logger = PrintLogger()
419else:
420logger = DisabledLogger()
421failures = getFailureData()
422if len(failures) < 1:
423print("Found 0 failures")
424return
Jeff Gastona7df0922021-06-02 17:39:47 -0400425locations = locate(failures)
426disable(locations)
Jeff Gastona7c77722023-11-02 15:15:59 -0400427commit()
Jeff Gastona7df0922021-06-02 17:39:47 -0400428
429if __name__ == "__main__":
430main()
431