diff --git a/docs/01-basic/report.md b/docs/01-basic/report.md new file mode 100644 index 0000000..d5ced96 --- /dev/null +++ b/docs/01-basic/report.md @@ -0,0 +1,60 @@ +## (Q1.1) + +### 1 + +在LogParser\Parser\LogFileParser.cs中 + +``` +using var csv = new CsvReader(logFile, config); + csv.Context.RegisterClassMap(); + + foreach (var logRecord in csv.GetRecords()) + +``` + +csv.GetRecords按照CSV规则解析各行。而每一列对应什么含义，也是在 LogFileParser 中通过 Index 指定的： + +``` + Map(m => m.LineNo).Index(0); + Map(m => m.Timestamp).Index(1); + Map(m => m.PodName).Index(2); + Map(m => m.Message).Index(3); + +``` + +### 2 + +``` + +using (var doc = JsonDocument.Parse(logRecord.Message)) + { + var root = doc.RootElement; + if (root.TryGetProperty("event", out var eventElement)) + +``` + +根据 eventElement 来判断类型 + +### 3 + +使用JsonSerializer.Deserialize。 + +同时设置 + +private static JsonSerializerOptions options = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.KebabCaseLower, + }; + +调用时将options传入，完成命名法转换 + +## （Q1.2） + ++ Dictionary KeyValueVisitor.Dump(LogEntry entry) ++ Dictionary CallLogEntry.Accept>(visitor) ++ Dictionary KeyValueVisitor.Visit(CallLogEntry entry) + +## （Q1.3） + +没有使用AI，时间花了大概三小时。比程设作业难。我认为我完成作业只是模仿示例完成了代码，还没有完全看懂整个架构。 + diff --git a/docs/02-multithreading/assets/localcli-normal.png b/docs/02-multithreading/assets/localcli-normal.png new file mode 100644 index 0000000..a5582bb Binary files /dev/null and b/docs/02-multithreading/assets/localcli-normal.png differ diff --git a/docs/02-multithreading/assets/localcli-robustness.png b/docs/02-multithreading/assets/localcli-robustness.png new file mode 100644 index 0000000..d7a5047 Binary files /dev/null and b/docs/02-multithreading/assets/localcli-robustness.png differ diff --git a/docs/02-multithreading/report.md b/docs/02-multithreading/report.md new file mode 100644 index 0000000..52b351d --- /dev/null +++ b/docs/02-multithreading/report.md @@ -0,0 +1,55 @@ +# T2.3 LocalCli 实现报告 + +## 实现功能 + +- 输入日志目录并创建分析器，也可以在运行期间切换目录。 +- 查看当前目录中的全部日志文件。 +- 设置并行度，分析指定的一个或多个日志文件。 +- 设置并行度，分析当前目录中的全部日志文件。 +- 查询日志文件的分析状态，并区分未分析、分析成功、分析失败和文件不存在四种情况。 +- 使用 `KeyValueVisitor.Dump` 输出成功解析的完整日志内容，并显示解析失败时的错误信息。 +- 校验目录、并行度和文件名等输入，捕获分析过程中产生的异常，避免程序因非法输入退出。 + +![LocalCli 完整功能截图](./assets/localcli-normal.png) + +## 鲁棒性测试截图 + +![LocalCli 鲁棒性测试截图](./assets/localcli-robustness.png) + +## Q2.1 + +1. 共享变量是队列 `_items` 和完成标志 `_isCompleted`；统一用 `lock (_items)` 保护，配合 `Wait/Pulse/PulseAll` 协调生产和消费。 + +2. `LogFileAnalyzer` 的目录、分析状态及两个字典由 `_syncRoot` 加锁保护；工作线程在锁外解析文件，完成后再加锁写入 `_analysisResults`。 + +3. 使用 `if` 遇到虚假唤醒会在空队列取值，导致异常或消费者提前退出；使用 `while` 可在每次唤醒后重新检查条件，确保队列非空或生产已结束。 + +## Q2.2 + +扫描代码是 `Directory.EnumerateFiles(directoryPath, "*.log", SearchOption.TopDirectoryOnly)`；递归时改用 `AllDirectories`，并以相对路径或完整路径作键，避免子目录同名文件冲突。 + +## Q2.3 + +使用AI，给AI的提示词之一为： + +``` + +public bool TryDequeue([NotNullWhen(true)] out T? item){ + lock (_items){ + while (_items.Count == 0 && !_isCompleted){ + Monitor.Wait(_items); + } + if (_items.Count > 0){ + item = _items.Dequeue(); + return true; + } + } + + item = default; + return false; +} +我现在这样写可能有什么问题？ + +``` + +我询问AI一些接口的用法，帮忙排查错误。目前未发现AI的解答有错误。我认为本节难度偏高 diff --git a/src/LocalCli/Program.cs b/src/LocalCli/Program.cs index 17b30db..0418637 100644 --- a/src/LocalCli/Program.cs +++ b/src/LocalCli/Program.cs @@ -27,6 +27,12 @@ public static void Main(string[] args) { return null; } + directory = directory.Trim(); + if (directory.Length == 0) + { + Console.WriteLine("Directory cannot be empty, please try again:"); + continue; + } try { if (!analyzer.ChangeDirectory(directory)) @@ -41,6 +47,11 @@ public static void Main(string[] args) Console.WriteLine("Directory illegal, please try again:"); continue; } + catch (Exception ex) + { + Console.WriteLine($"Failed to open directory: {ex.Message}"); + continue; + } } return analyzer; } @@ -112,22 +123,138 @@ 6. Exit. private static void ShowLogFiles(LogFileAnalyzer analyzer) { - throw new NotImplementedException("T2.3"); + var files = analyzer.GetLogFiles(); + Console.WriteLine($"[{string.Join(", ", files)}]"); } private static void AnalyzeFiles(LogFileAnalyzer analyzer) { - throw new NotImplementedException("T2.3"); + try + { + var degreeOfParallelism = ReadDegreeOfParallelism(); + var fileNames = ReadFileNames(); + + analyzer.AnalyzeFiles(degreeOfParallelism, fileNames); + Console.WriteLine($"Analysis completed: [{string.Join(", ", fileNames)}]"); + } + catch (Exception ex) + { + Console.WriteLine($"Analysis failed: {ex.Message}"); + } } private static void AnalyzeAll(LogFileAnalyzer analyzer) { - throw new NotImplementedException("T2.3"); + try + { + var degreeOfParallelism = ReadDegreeOfParallelism(); + var fileNames = analyzer.GetLogFiles(); + + analyzer.AnalyzeAll(degreeOfParallelism); + Console.WriteLine($"Analysis completed: [{string.Join(", ", fileNames)}]"); + } + catch (Exception ex) + { + Console.WriteLine($"Analysis failed: {ex.Message}"); + } } private static void GetAnalysisResult(LogFileAnalyzer analyzer) { - throw new NotImplementedException("T2.3"); + Console.WriteLine("Please input log file name:"); + var input = Console.ReadLine(); + if (input is null) + { + return; + } + + var fileName = input.Trim(); + if (fileName.Length == 0) + { + Console.WriteLine("File name cannot be empty."); + return; + } + + if (!analyzer.TryGetAnalysisResult(fileName, out var result) || result is null) + { + Console.WriteLine($"File {fileName} does not exist."); + return; + } + + switch (result.State) + { + case AnalysisState.NotAnalyzed: + Console.WriteLine($"File {fileName} has not been analyzed yet."); + break; + + case AnalysisState.Failed: + Console.WriteLine( + $"Analysis failed for {fileName}: {result.ErrorMessage ?? "Unknown error"}"); + break; + + case AnalysisState.Succeeded: + Console.WriteLine($"Analysis result for {fileName}:"); + var visitor = new KeyValueVisitor(); + foreach (var entry in result.Entries) + { + var values = visitor.Dump(entry); + Console.WriteLine(string.Join(", ", + values.Select(pair => $"{pair.Key}: {pair.Value}"))); + } + break; + + default: + Console.WriteLine($"Unknown analysis state for {fileName}: {result.State}"); + break; + } + } + + private static int ReadDegreeOfParallelism() + { + while (true) + { + Console.WriteLine("Please input degree of parallelism:"); + var input = Console.ReadLine(); + if (input is null) + { + throw new EndOfStreamException("Input ended."); + } + + if (int.TryParse(input.Trim(), out var degreeOfParallelism) + && degreeOfParallelism >= 0) + { + return degreeOfParallelism; + } + + Console.WriteLine("Invalid degree of parallelism, please try again:"); + } + } + + private static List ReadFileNames() + { + while (true) + { + Console.WriteLine("Please input log file names (comma separated):"); + var input = Console.ReadLine(); + if (input is null) + { + throw new EndOfStreamException("Input ended."); + } + + var fileNames = input + .Split(',') + .Select(fileName => fileName.Trim()) + .Where(fileName => fileName.Length > 0) + .Distinct(StringComparer.Ordinal) + .ToList(); + + if (fileNames.Count > 0) + { + return fileNames; + } + + Console.WriteLine("No log file names provided, please try again:"); + } } } } diff --git a/src/LogAnalyzer/LogFileAnalyzer.cs b/src/LogAnalyzer/LogFileAnalyzer.cs index c3e7691..c411933 100644 --- a/src/LogAnalyzer/LogFileAnalyzer.cs +++ b/src/LogAnalyzer/LogFileAnalyzer.cs @@ -137,11 +137,7 @@ public void AnalyzeFiles(int degreeOfParallelism, IEnumerable fileNames) } } fileList = fileNameList.Select(fileName => _logFiles[fileName]).ToList(); - - /* - * Set _isAnalyzing - */ - // TODO: T2.2 + _isAnalyzing = true; } try @@ -150,11 +146,10 @@ public void AnalyzeFiles(int degreeOfParallelism, IEnumerable fileNames) } finally { - /* - * Unset _isAnalyzing - * Remember to lock _syncRoot to prevent data race - */ - // TODO: T2.2 + lock (_syncRoot) + { + _isAnalyzing = false; + } } } @@ -165,11 +160,16 @@ private void RunWorkers(int degreeOfParallelism, IReadOnlyList fileLis { foreach (var file in fileList) { - /* - * Filter unparsed files. - * If there is an unknown file, throw System.InvalidOperationException. - */ - throw new NotImplementedException("TODO: T2.2"); + if (!_analysisResults.TryGetValue(file.Name, out var result)) + { + throw new InvalidOperationException( + $"File '{file.Name}' is not registered in the analyzer." + ); + } + if (result.State == AnalysisState.NotAnalyzed) + { + logFilesToParse.Add(file); + } } } @@ -180,10 +180,12 @@ private void RunWorkers(int degreeOfParallelism, IReadOnlyList fileLis var queue = new WorkQueue(); - /* - * Enqueue log files - */ - // TODO: T2.2 + foreach (var file in logFilesToParse) + { + queue.Enqueue(file); + } + + queue.CompleteAdding(); degreeOfParallelism = Math.Max(Math.Min(degreeOfParallelism, logFilesToParse.Count), 1); var workers = new Thread[degreeOfParallelism]; @@ -191,16 +193,20 @@ private void RunWorkers(int degreeOfParallelism, IReadOnlyList fileLis { int workerId = i; string threadName = $"log-analyzer-worker-{workerId}"; - /* - * Create and start threads to run `WorkerMain` - */ - // TODO: T2.2 + + workers[i] = new Thread(() => WorkerMain(workerId, queue)) + { + Name = threadName, + IsBackground = true + }; + + workers[i].Start(); } - /* - * Wait for (join) all threads to end - */ - // TODO: T2.2 + foreach (var worker in workers) + { + worker.Join(); + } } private void WorkerMain(int workerId, WorkQueue queue) @@ -212,20 +218,36 @@ private void WorkerMain(int workerId, WorkQueue queue) AnalysisResult result; try { - // Parse file - throw new NotImplementedException("TODO: T2.2"); + using var reader = new StreamReader(file.FullName); + + var entries = parser.Parse(reader).ToList(); + + result = new AnalysisResult( + FileName: file.Name, + FullName: file.FullName, + State: AnalysisState.Succeeded, + Entries: entries, + ErrorMessage: null, + WorkerId: workerId + ); + } catch (Exception ex) { - // Save exception message to result - throw new NotImplementedException("TODO: T2.2"); + result = new AnalysisResult( + FileName: file.Name, + FullName: file.FullName, + State: AnalysisState.Failed, + Entries: Array.Empty(), + ErrorMessage: ex.Message, + WorkerId: workerId + ); } - /* - * Save parse result. - * [!Important] Remember to lock _syncRoot to prevent data race. - */ - throw new NotImplementedException("TODO: T2.2"); + lock (_syncRoot) + { + _analysisResults[file.Name] = result; + } } } } diff --git a/src/LogAnalyzer/WorkQueue.cs b/src/LogAnalyzer/WorkQueue.cs index 23055a5..eba4f4f 100644 --- a/src/LogAnalyzer/WorkQueue.cs +++ b/src/LogAnalyzer/WorkQueue.cs @@ -20,17 +20,50 @@ public bool IsCompleted public void Enqueue(T item) { - throw new NotImplementedException("TODO: T2.1"); + if (item is null) + { + throw new ArgumentNullException(nameof(item)); + } + + lock (_items) + { + if (_isCompleted) + { + throw new InvalidOperationException( + "Cannot enqueue after adding has completed."); + } + + _items.Enqueue(item); + Monitor.Pulse(_items); + } } public bool TryDequeue([NotNullWhen(true)] out T? item) { - throw new NotImplementedException("TODO: T2.1"); + lock (_items) + { + while (_items.Count == 0 && !_isCompleted) + { + Monitor.Wait(_items); + } + if (_items.Count > 0) + { + item = _items.Dequeue()!; + return true; + } + } + + item = default; + return false; } public void CompleteAdding() { - throw new NotImplementedException("TODO: T2.1"); + lock (_items) + { + _isCompleted = true; + Monitor.PulseAll(_items); + } } } } diff --git a/src/LogParser/Models/LogEntries.cs b/src/LogParser/Models/LogEntries.cs index 69edbc0..e4e9bbc 100644 --- a/src/LogParser/Models/LogEntries.cs +++ b/src/LogParser/Models/LogEntries.cs @@ -54,7 +54,7 @@ public sealed record RequestLogEntry( { public override TResult Accept(ILogEntryVisitor visitor) { - throw new NotImplementedException("TODO: T1.2"); + return visitor.Visit(this); } } @@ -69,7 +69,7 @@ public sealed record InternalLogEntry( { public override TResult Accept(ILogEntryVisitor visitor) { - throw new NotImplementedException("TODO: T1.2"); + return visitor.Visit(this); } } diff --git a/src/LogParser/Parser/LineParser.cs b/src/LogParser/Parser/LineParser.cs index 0475f6b..82a964a 100644 --- a/src/LogParser/Parser/LineParser.cs +++ b/src/LogParser/Parser/LineParser.cs @@ -16,8 +16,8 @@ public static LogEntry ParseLine(LogRecord logRecord) return eventElement.GetString() switch { "call" => LineParser.CreateCall(logRecord), - "request" => throw new NotImplementedException("TODO: T1.2"), - "internal" => throw new NotImplementedException("TODO: T1.2"), + "request" => LineParser.CreateRequest(logRecord), + "internal" => LineParser.CreateInternal(logRecord), _ => throw new FormatException($"Unknown event type: {eventElement.GetString()} in log message: {logRecord.Message}") }; } @@ -50,12 +50,40 @@ private static LogEntry CreateCall(LogRecord logRecord) private static LogEntry CreateRequest(LogRecord logRecord) { - throw new NotImplementedException("TODO: T1.2"); + var requestMessage = JsonSerializer.Deserialize(logRecord.Message, options) + ?? throw new FormatException($"Failed to deserialize call message: {logRecord.Message}"); + return new RequestLogEntry( + LineNo: logRecord.LineNo, + Timestamp: DateTimeOffset.Parse(logRecord.Timestamp), + PodName: logRecord.PodName, + Severity: ParseSeverity(requestMessage.Severity), + RequestId: requestMessage.RequestId, + Method: requestMessage.Method, + Path: requestMessage.Path, + StatusCode: requestMessage.StatusCode + ); } private static LogEntry CreateInternal(LogRecord logRecord) { - throw new NotImplementedException("TODO: T1.2"); + var internalMessage = JsonSerializer.Deserialize(logRecord.Message, options) + ?? throw new FormatException($"Failed to deserialize call message: {logRecord.Message}"); + var Exception = internalMessage.Exception; + var idx = Exception.IndexOf(":"); + if (idx < 0) + { + throw new FormatException($"Invalid exception format: {internalMessage.Exception}"); + } + var ExceptionName = Exception.Substring(0,idx); + var ExceptionMessage = Exception.Substring(idx+2); + return new InternalLogEntry( + LineNo: logRecord.LineNo, + Timestamp: DateTimeOffset.Parse(logRecord.Timestamp), + PodName: logRecord.PodName, + Severity: ParseSeverity(internalMessage.Severity), + ExceptionName: ExceptionName, + ExceptionMessage: ExceptionMessage + ); } private static LogSeverity ParseSeverity(string severity) @@ -77,11 +105,18 @@ private record CallMessage( ); private record RequestMessage( - // TODO: T1.2 + [property: JsonRequired] string Severity, + [property: JsonRequired] string RequestId, + [property: JsonRequired] string Method, + [property: JsonRequired] string Path, + [property: JsonRequired] int StatusCode + ); private record InternalMessage( - // TODO: T1.2 + [property: JsonRequired] string Severity, + [property: JsonRequired] string Exception + ); } } diff --git a/src/LogParser/Visitors/KeyValueVisitor.cs b/src/LogParser/Visitors/KeyValueVisitor.cs index e5ceba2..c4ee145 100644 --- a/src/LogParser/Visitors/KeyValueVisitor.cs +++ b/src/LogParser/Visitors/KeyValueVisitor.cs @@ -26,12 +26,32 @@ public Dictionary Visit(CallLogEntry entry) public Dictionary Visit(RequestLogEntry entry) { - throw new NotImplementedException("TODO: T1.3"); + return new Dictionary + { + ["LineNo"] = entry.LineNo.ToString(), + ["Timestamp"] = entry.Timestamp.ToString("O"), + ["PodName"] = entry.PodName, + ["Severity"] = entry.Severity.ToString(), + ["EventType"] = entry.EventType.ToString(), + ["RequestId"] = entry.RequestId, + ["Method"] = entry.Method, + ["Path"] = entry.Path, + ["StatusCode"] = entry.StatusCode.ToString() + }; } public Dictionary Visit(InternalLogEntry entry) { - throw new NotImplementedException("TODO: T1.3"); + return new Dictionary + { + ["LineNo"] = entry.LineNo.ToString(), + ["Timestamp"] = entry.Timestamp.ToString("O"), + ["PodName"] = entry.PodName, + ["Severity"] = entry.Severity.ToString(), + ["EventType"] = entry.EventType.ToString(), + ["ExceptionName"] = entry.ExceptionName, + ["ExceptionMessage"] = entry.ExceptionMessage + }; } } }