From a4058d6c4829a9fc6077ac1203e2dec3b34c5b2c Mon Sep 17 00:00:00 2001 From: Erick Camacho Date: Mon, 29 Nov 2021 10:58:34 +0100 Subject: [PATCH 1/2] Add flag to truncate number of issues reported in a task Signed-off-by: Erick Camacho --- README.md | 3 +- .../XCLogParser/commands/ActionOptions.swift | 8 ++- .../XCLogParser/commands/CommandHandler.swift | 3 +- Sources/XCLogParser/commands/Version.swift | 2 +- .../IDEActivityLogSection+Builders.swift | 46 +++++++++++++++++ .../XCLogParser/parser/Notice+Parser.swift | 33 +++++++++++- .../XCLogParser/parser/ParserBuildSteps.swift | 11 +++- .../commands/ParseCommand.swift | 11 +++- Tests/XCLogParserTests/ParserTests.swift | 50 +++++++++++++++++-- 9 files changed, 154 insertions(+), 13 deletions(-) create mode 100644 Sources/XCLogParser/parser/IDEActivityLogSection+Builders.swift diff --git a/README.md b/README.md index 0f93f92..2af6a58 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ An example output has been omitted for brevity since it can contain a lot of inf Parses the build information from a `xcactivitylog` and converts it into different representations such as a [JSON file](#JSON-Reporter), [flat JSON file](#FlatJson-Reporter), [summary JSON file](#SummaryJson-Reporter), [issues JSON file](#Issues-Reporter), [Chrome Tracer file](#ChromeTracer-Reporter) or a static [HTML page](#HTML-Reporter). -This command supports parsing additional data if some flags are passed to Xcode/xcodebuild: +This command supports parsing additional data if some flags are passed to Xcode/xcodebuild: 1. `swiftc` reported compilation times. For using that feature, you need to build your project with the options `-Xfrontend -debug-time-expression-type-checking` and `-Xfrontend -debug-time-function-bodies`. 2. ld64's statistics output. The statistics info can be generated by adding `-Xlinker -print_statistics` to Xcode's "Other Linker Flags" and it's useful for tracking linking time regression. @@ -161,6 +161,7 @@ Example output available in the [reporters](#reporters) section. | `--machine_name` | If specified, the machine name will be used to create the `buildIdentifier`. If it is not specified, the host name will be used. | No | | `--omit_warnings` | Omit the warnings details in the final report. This is useful if there are too many of them and the report's size is too big with them. | No | | `--omit_notes` | Omit the notes details in the final report. This is useful if there are too many of them and the report's size is too big with them. | No | + | `--trunc_large_issues` | If an individual task has more than a 100 issues (Warnings, notes, errors) it truncates them to be 100. This is useful to reduce the amount of memory used. | No | >No *: One of `--file`, `--project`, `--workspace`, `--xcodeproj` parameters is required. diff --git a/Sources/XCLogParser/commands/ActionOptions.swift b/Sources/XCLogParser/commands/ActionOptions.swift index 300bb5b..6cd81d7 100644 --- a/Sources/XCLogParser/commands/ActionOptions.swift +++ b/Sources/XCLogParser/commands/ActionOptions.swift @@ -52,6 +52,9 @@ public struct ActionOptions { /// If true, the parse command won't add the Notes details to the output. public let omitNotesDetails: Bool + /// If true, tasks with more than a 100 issues (warnings, errors, notes) will be truncated to a 100 + public let truncLargeIssues: Bool + public init(reporter: Reporter, outputPath: String, redacted: Bool, @@ -59,7 +62,9 @@ public struct ActionOptions { machineName: String? = nil, rootOutput: String = "", omitWarningsDetails: Bool = false, - omitNotesDetails: Bool = false) { + omitNotesDetails: Bool = false, + truncLargeIssues: Bool = false + ) { self.reporter = reporter self.outputPath = outputPath self.redacted = redacted @@ -68,5 +73,6 @@ public struct ActionOptions { self.rootOutput = rootOutput self.omitWarningsDetails = omitWarningsDetails self.omitNotesDetails = omitNotesDetails + self.truncLargeIssues = truncLargeIssues } } diff --git a/Sources/XCLogParser/commands/CommandHandler.swift b/Sources/XCLogParser/commands/CommandHandler.swift index 8a8a52e..dab1c2d 100644 --- a/Sources/XCLogParser/commands/CommandHandler.swift +++ b/Sources/XCLogParser/commands/CommandHandler.swift @@ -65,7 +65,8 @@ public struct CommandHandler { let buildParser = ParserBuildSteps(machineName: options.machineName, omitWarningsDetails: options.omitWarningsDetails, - omitNotesDetails: options.omitNotesDetails) + omitNotesDetails: options.omitNotesDetails, + truncLargeIssues: options.truncLargeIssues) let buildSteps = try buildParser.parse(activityLog: activityLog) let reporterOutput = ReporterOutputFactory.makeReporterOutput(path: options.outputPath) let logReporter = options.reporter.makeLogReporter() diff --git a/Sources/XCLogParser/commands/Version.swift b/Sources/XCLogParser/commands/Version.swift index 7973f04..4d80203 100644 --- a/Sources/XCLogParser/commands/Version.swift +++ b/Sources/XCLogParser/commands/Version.swift @@ -21,6 +21,6 @@ import Foundation public struct Version { - public static let current = "0.2.30" + public static let current = "0.2.31" } diff --git a/Sources/XCLogParser/parser/IDEActivityLogSection+Builders.swift b/Sources/XCLogParser/parser/IDEActivityLogSection+Builders.swift new file mode 100644 index 0000000..309e356 --- /dev/null +++ b/Sources/XCLogParser/parser/IDEActivityLogSection+Builders.swift @@ -0,0 +1,46 @@ +// Copyright (c) 2021 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +extension IDEActivityLogSection { + + func with(messages newMessages: [IDEActivityLogMessage]) -> IDEActivityLogSection { + return IDEActivityLogSection(sectionType: self.sectionType, + domainType: self.domainType, + title: self.title, + signature: self.signature, + timeStartedRecording: self.timeStartedRecording, + timeStoppedRecording: self.timeStoppedRecording, + subSections: self.subSections, + text: self.text, + messages: newMessages, + wasCancelled: self.wasCancelled, + isQuiet: self.isQuiet, + wasFetchedFromCache: self.wasFetchedFromCache, + subtitle: self.subtitle, + location: self.location, + commandDetailDesc: self.commandDetailDesc, + uniqueIdentifier: self.uniqueIdentifier, + localizedResultString: self.localizedResultString, + xcbuildSignature: self.xcbuildSignature, + unknown: self.unknown) + } + +} diff --git a/Sources/XCLogParser/parser/Notice+Parser.swift b/Sources/XCLogParser/parser/Notice+Parser.swift index 183465a..50e1ce9 100644 --- a/Sources/XCLogParser/parser/Notice+Parser.swift +++ b/Sources/XCLogParser/parser/Notice+Parser.swift @@ -27,9 +27,16 @@ extension Notice { /// For CLANG warnings, it parses the `IDEActivityLogSection` text property looking for a *-W-warning-name* pattern /// - parameter logSection: An `IDEActivityLogSection` /// - parameter forType: The `DetailStepType` of the logSection + /// - parameter truncLargeIssues: If true, if a task have more than 100 `Notice`, will be truncated to 100 /// - returns: An Array of `Notice` - public static func parseFromLogSection(_ logSection: IDEActivityLogSection, forType type: DetailStepType) + public static func parseFromLogSection(_ logSection: IDEActivityLogSection, + forType type: DetailStepType, + truncLargeIssues: Bool) -> [Notice] { + var logSection = logSection + if truncLargeIssues && logSection.messages.count > 100 { + logSection = self.logSectionWithTruncatedIssues(logSection: logSection) + } // we look for clangWarnings parsing the text of the logSection let clangWarningsFlags = self.parseClangWarningFlags(text: logSection.text) let clangWarnings = self.parseClangWarnings(clangFlags: clangWarningsFlags, logSection: logSection) @@ -187,4 +194,28 @@ extension Notice { } return false } + + private static func logSectionWithTruncatedIssues(logSection: IDEActivityLogSection) -> IDEActivityLogSection { + let issuesKept = min(99, logSection.messages.count) + var truncatedMessages = Array(logSection.messages[0.. IDEActivityLogMessage { + let title = "Warning: \(logSection.messages.count - issuesKept) issues were truncated" + return IDEActivityLogMessage(title: title, + shortTitle: "", + timeEmitted: 0, + rangeEndInSectionText: 0, + rangeStartInSectionText: 0, + subMessages: [], + severity: 0, + type: "", + location: DVTDocumentLocation(documentURLString: "", timestamp: 0), + categoryIdent: "Warning", + secondaryLocations: [], + additionalDescription: "") + } } diff --git a/Sources/XCLogParser/parser/ParserBuildSteps.swift b/Sources/XCLogParser/parser/ParserBuildSteps.swift index db8427b..4149efa 100644 --- a/Sources/XCLogParser/parser/ParserBuildSteps.swift +++ b/Sources/XCLogParser/parser/ParserBuildSteps.swift @@ -42,6 +42,10 @@ public final class ParserBuildSteps { /// Usefult to save space. let omitNotesDetails: Bool + /// If true, tasks with more than a 100 issues will be + /// truncated to have only 100 + let truncLargeIssues: Bool + public lazy var dateFormatter: DateFormatter = { let formatter = DateFormatter() formatter.timeZone = TimeZone(abbreviation: "UTC") @@ -79,9 +83,11 @@ public final class ParserBuildSteps { /// for the log. If `nil`, the host name will be used instead. /// - parameter omitWarningsDetails: if true, the Warnings won't be parsed /// - parameter omitNotesDetails: if true, the Notes won't be parsed + /// - parameter truncLargeIssues: if true, tasks with more than a 100 issues will be truncated to have a 100 public init(machineName: String? = nil, omitWarningsDetails: Bool, - omitNotesDetails: Bool) { + omitNotesDetails: Bool, + truncLargeIssues: Bool) { if let machineName = machineName { self.machineName = machineName } else { @@ -89,6 +95,7 @@ public final class ParserBuildSteps { } self.omitWarningsDetails = omitWarningsDetails self.omitNotesDetails = omitNotesDetails + self.truncLargeIssues = truncLargeIssues } /// Parses the content from an Xcode log into a `BuildStep` @@ -298,7 +305,7 @@ public final class ParserBuildSteps { private func parseWarningsAndErrorsFromLogSection(_ logSection: IDEActivityLogSection, forType type: DetailStepType) -> [String: [Notice]]? { - let notices = Notice.parseFromLogSection(logSection, forType: type) + let notices = Notice.parseFromLogSection(logSection, forType: type, truncLargeIssues: truncLargeIssues) return ["warnings": notices.getWarnings(), "errors": notices.getErrors(), "notes": notices.getNotes()] diff --git a/Sources/XCLogParserApp/commands/ParseCommand.swift b/Sources/XCLogParserApp/commands/ParseCommand.swift index 7bb4e2d..3b9124f 100644 --- a/Sources/XCLogParserApp/commands/ParseCommand.swift +++ b/Sources/XCLogParserApp/commands/ParseCommand.swift @@ -121,6 +121,14 @@ struct ParseCommand: ParsableCommand { """) var omitNotes: Bool = false + @Flag(name: .customLong("trunc_large_issues"), + help: """ + If present, for tasks with more than a 100 issues (warnings, notes or errors) + Those will be truncated to a 100. + Useful to reduce the amount of memory used and the size of the report. + """) + var truncLargeIssues: Bool = false + mutating func validate() throws { if !hasValidLogOptions() { throw ValidationError(""" @@ -163,7 +171,8 @@ struct ParseCommand: ParsableCommand { machineName: machineName, rootOutput: rootOutput ?? "", omitWarningsDetails: omitWarnings, - omitNotesDetails: omitNotes) + omitNotesDetails: omitNotes, + truncLargeIssues: truncLargeIssues) let action = Action.parse(options: actionOptions) let command = Command(logOptions: logOptions, action: action) try commandHandler.handle(command: command) diff --git a/Tests/XCLogParserTests/ParserTests.swift b/Tests/XCLogParserTests/ParserTests.swift index 7dd0449..8d6c916 100644 --- a/Tests/XCLogParserTests/ParserTests.swift +++ b/Tests/XCLogParserTests/ParserTests.swift @@ -24,7 +24,8 @@ import XCTest class ParserTests: XCTestCase { let parser = ParserBuildSteps(omitWarningsDetails: false, - omitNotesDetails: false) + omitNotesDetails: false, + truncLargeIssues: false) func testDateFormatterUsesJSONFormat() { let jsonDateString = "2014-09-27T12:30:00.450000Z" @@ -42,7 +43,8 @@ class ParserTests: XCTestCase { let timestamp = Date().timeIntervalSinceNow let parser = ParserBuildSteps(machineName: machineName, omitWarningsDetails: false, - omitNotesDetails: false) + omitNotesDetails: false, + truncLargeIssues: false) let fakeMainSection = IDEActivityLogSection(sectionType: 1, domainType: "", title: "Main", @@ -70,7 +72,8 @@ class ParserTests: XCTestCase { if let hostName = Host.current().localizedName { let parserNoMachineName = ParserBuildSteps(machineName: nil, omitWarningsDetails: false, - omitNotesDetails: false) + omitNotesDetails: false, + truncLargeIssues: false) let buildStepNoMachineName = try parserNoMachineName.parse(activityLog: fakeActivityLog) XCTAssertEqual("\(hostName)_\(uniqueIdentifier)", buildStepNoMachineName.buildIdentifier) } @@ -449,7 +452,8 @@ note: use 'updatedDoSomething' instead\r doSomething()\r ^~~~~~~~~~~\r andText: "This is deprecated, [-Wdeprecated-declarations]", loc: textDocumentLocation) let parser = ParserBuildSteps(omitWarningsDetails: true, - omitNotesDetails: false) + omitNotesDetails: false, + truncLargeIssues: false) let build = try parser.parse(activityLog: fakeLog) XCTAssertEqual(0, build.warnings?.count ?? 0, "Warnings should be empty") XCTAssertEqual(1, build.warningCount, "Number of warnings should be reported") @@ -484,11 +488,47 @@ note: use 'updatedDoSomething' instead\r doSomething()\r ^~~~~~~~~~~\r andText: "Log", loc: textDocumentLocation) let parser = ParserBuildSteps(omitWarningsDetails: false, - omitNotesDetails: true) + omitNotesDetails: true, + truncLargeIssues: false) let build = try parser.parse(activityLog: fakeLog) XCTAssertEqual(0, build.notes?.count ?? 0, "Notes should be empty") } + func testParseTruncateLargeIssues() throws { + let timestamp = Date().timeIntervalSinceReferenceDate + let textDocumentLocation = DVTTextDocumentLocation(documentURLString: "file://project/file.swift", + timestamp: timestamp, + startingLineNumber: 10, + startingColumnNumber: 11, + endingLineNumber: 12, + endingColumnNumber: 13, + characterRangeEnd: 14, + characterRangeStart: 15, + locationEncoding: 16) + let aThousandWarnings = (0...999).map { index in + IDEActivityLogMessage(title: "Swift Compiler Warning", + shortTitle: "", + timeEmitted: timestamp, + rangeEndInSectionText: 18446744073709551615, + rangeStartInSectionText: 0, + subMessages: [], + severity: 1, + type: "com.apple.dt.IDE.diagnostic", + location: textDocumentLocation, + categoryIdent: "", + secondaryLocations: [], + additionalDescription: "") + } + let fakeLog = getFakeIDEActivityLogWithMessages(aThousandWarnings, + andText: "Swift Compiler Warning", + loc: textDocumentLocation) + let parser = ParserBuildSteps(omitWarningsDetails: false, + omitNotesDetails: false, + truncLargeIssues: true) + let build = try parser.parse(activityLog: fakeLog) + XCTAssertEqual(100, build.warnings?.count ?? 0, "Warnings should be truncated up to 100") + } + // swiftlint:disable line_length let commandDetailSwiftSteps = """ CompileSwift normal x86_64 (in target 'Alamofire' from project 'Pods') From 1785d71b1f8f8371dafbd73b3332e379ba24ab75 Mon Sep 17 00:00:00 2001 From: Erick Camacho Date: Mon, 29 Nov 2021 11:19:49 +0100 Subject: [PATCH 2/2] Fix Swiftlint violation Signed-off-by: Erick Camacho --- Tests/XCLogParserTests/ParserTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/XCLogParserTests/ParserTests.swift b/Tests/XCLogParserTests/ParserTests.swift index 8d6c916..3af5682 100644 --- a/Tests/XCLogParserTests/ParserTests.swift +++ b/Tests/XCLogParserTests/ParserTests.swift @@ -505,7 +505,7 @@ note: use 'updatedDoSomething' instead\r doSomething()\r ^~~~~~~~~~~\r characterRangeEnd: 14, characterRangeStart: 15, locationEncoding: 16) - let aThousandWarnings = (0...999).map { index in + let aThousandWarnings = (0...999).map { _ in IDEActivityLogMessage(title: "Swift Compiler Warning", shortTitle: "", timeEmitted: timestamp,