From 147c95a48a888cdbc2dfcc6707e7041feea5ed37 Mon Sep 17 00:00:00 2001 From: Roman Pytkov Date: Wed, 5 Nov 2025 01:18:12 +0300 Subject: [PATCH] Enables code execution within a Docker sandbox Adds a Docker-based execution sandbox to enhance the security and resource management of code execution. This change introduces: - New execution services for C#, C++, Java, Kotlin, and Python that utilize the sandbox. - Configuration options for enabling/disabling the sandbox and specifying Docker images for different languages. - Batch execution support for C++ to improve the efficiency of generating answer files. - Docker CLI installation in the worker's Dockerfile. The sandbox provides improved isolation and resource control during code execution, preventing potential security vulnerabilities and resource exhaustion. --- src/LiquidCode.Tester.Worker/Dockerfile | 2 + src/LiquidCode.Tester.Worker/Program.cs | 3 + .../Services/AnswerGenerationService.cs | 104 ++++- .../Services/CSharpExecutionService.cs | 108 +---- .../Services/CppExecutionService.cs | 251 ++++++---- .../Services/JavaExecutionService.cs | 112 +---- .../Services/KotlinExecutionService.cs | 111 +---- .../Services/PackageParserService.cs | 78 +++- .../Services/PythonExecutionService.cs | 114 +---- .../Services/Sandbox/ExecutionResultMapper.cs | 43 ++ .../Services/Sandbox/ExecutionSandbox.cs | 435 ++++++++++++++++++ .../Sandbox/ExecutionSandboxModels.cs | 27 ++ .../Services/Sandbox/IExecutionSandbox.cs | 9 + .../Services/Sandbox/SandboxOptions.cs | 21 + .../appsettings.Development.json | 3 + src/LiquidCode.Tester.Worker/appsettings.json | 14 + .../ExecutionServiceFactoryTests.cs | 2 + .../LiquidCode.Tester.Worker.Tests.csproj | 1 + .../PolygonPackageIntegrationTests.cs | 92 +++- 19 files changed, 1023 insertions(+), 507 deletions(-) create mode 100644 src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionResultMapper.cs create mode 100644 src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionSandbox.cs create mode 100644 src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionSandboxModels.cs create mode 100644 src/LiquidCode.Tester.Worker/Services/Sandbox/IExecutionSandbox.cs create mode 100644 src/LiquidCode.Tester.Worker/Services/Sandbox/SandboxOptions.cs diff --git a/src/LiquidCode.Tester.Worker/Dockerfile b/src/LiquidCode.Tester.Worker/Dockerfile index f1f444f..86da30a 100644 --- a/src/LiquidCode.Tester.Worker/Dockerfile +++ b/src/LiquidCode.Tester.Worker/Dockerfile @@ -40,6 +40,8 @@ RUN apt-get update && \ # Python python3 \ python3-pip \ + # Docker CLI for nested sandboxing + docker.io \ # Kotlin compiler wget \ unzip \ diff --git a/src/LiquidCode.Tester.Worker/Program.cs b/src/LiquidCode.Tester.Worker/Program.cs index c97a24c..ed4fa4e 100644 --- a/src/LiquidCode.Tester.Worker/Program.cs +++ b/src/LiquidCode.Tester.Worker/Program.cs @@ -1,4 +1,5 @@ using LiquidCode.Tester.Worker.Services; +using LiquidCode.Tester.Worker.Services.Sandbox; var builder = WebApplication.CreateBuilder(args); @@ -15,6 +16,8 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.Configure(builder.Configuration.GetSection("Sandbox")); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/src/LiquidCode.Tester.Worker/Services/AnswerGenerationService.cs b/src/LiquidCode.Tester.Worker/Services/AnswerGenerationService.cs index a9c1095..8e43e8a 100644 --- a/src/LiquidCode.Tester.Worker/Services/AnswerGenerationService.cs +++ b/src/LiquidCode.Tester.Worker/Services/AnswerGenerationService.cs @@ -71,12 +71,94 @@ public class AnswerGenerationService return false; } - _logger.LogInformation("Main solution compiled successfully"); + _logger.LogInformation("Main solution compiled successfully to {Executable}", compilationResult.ExecutablePath); - // Generate answers for each test - int generatedCount = 0; - for (int i = 0; i < inputFilePaths.Count; i++) + var totalTests = inputFilePaths.Count; + if (totalTests == 0) { + return false; + } + + var problemTimeLimit = descriptor.TimeLimitMs > 0 ? descriptor.TimeLimitMs : 1000; + var answerTimeLimit = Math.Max(problemTimeLimit * 5, 10000); + var executionMemoryLimit = descriptor.MemoryLimitMb > 0 ? descriptor.MemoryLimitMb * 2 : 512; + + var completedIndices = new HashSet(); + var generatedCount = 0; + + if (executionService is CppExecutionService cppExecutionService && totalTests > 1) + { + var batchResult = await cppExecutionService.ExecuteBatchAsync( + compilationResult.ExecutablePath!, + workingDirectory, + inputFilePaths, + answerFilePaths, + answerTimeLimit, + executionMemoryLimit).ConfigureAwait(false); + + if (batchResult is not null) + { + for (var i = 0; i < totalTests; i++) + { + if (File.Exists(answerFilePaths[i]) && new FileInfo(answerFilePaths[i]).Length > 0) + { + completedIndices.Add(i); + generatedCount++; + } + } + + if (batchResult.Success && completedIndices.Count == totalTests) + { + _logger.LogInformation( + batchResult.UsedSandbox + ? "Generated answers for {Count} tests via sandbox batch execution" + : "Generated answers for {Count} tests via local batch execution", + totalTests); + return true; + } + + if (completedIndices.Count > 0 && completedIndices.Count < totalTests) + { + _logger.LogInformation( + "{Mode} produced {Generated}/{Total} answers; retrying remaining {Remaining} individually", + batchResult.UsedSandbox ? "Sandbox batch execution" : "Local batch execution", + completedIndices.Count, + totalTests, + totalTests - completedIndices.Count); + } + + if (!batchResult.Success) + { + var error = string.IsNullOrWhiteSpace(batchResult.ErrorMessage) + ? batchResult.StandardError + : batchResult.ErrorMessage; + + var mode = batchResult.UsedSandbox ? "Sandbox batch execution" : "Local batch execution"; + + if (!string.IsNullOrWhiteSpace(error)) + { + _logger.LogWarning("{Mode} exited with code {ExitCode}: {Message}", mode, batchResult.ExitCode, error.Trim()); + } + else + { + _logger.LogWarning("{Mode} exited with code {ExitCode}", mode, batchResult.ExitCode); + } + } + + if (completedIndices.Count == totalTests) + { + return generatedCount > 0; + } + } + } + + for (int i = 0; i < totalTests; i++) + { + if (completedIndices.Contains(i)) + { + continue; + } + var inputPath = inputFilePaths[i]; var answerPath = answerFilePaths[i]; @@ -87,14 +169,13 @@ public class AnswerGenerationService } _logger.LogDebug("Generating answer {Index}/{Total}: {AnswerPath}", - i + 1, inputFilePaths.Count, answerPath); + i + 1, totalTests, answerPath); - // Execute solution with input var executionResult = await executionService.ExecuteAsync( compilationResult.ExecutablePath!, inputPath, - descriptor.TimeLimitMs * 2, // Give extra time for answer generation - descriptor.MemoryLimitMb * 2); + answerTimeLimit, + executionMemoryLimit).ConfigureAwait(false); if (!executionResult.Success || executionResult.RuntimeError) { @@ -103,21 +184,20 @@ public class AnswerGenerationService continue; } - // Save output as answer file var answerDir = Path.GetDirectoryName(answerPath); if (!string.IsNullOrEmpty(answerDir) && !Directory.Exists(answerDir)) { Directory.CreateDirectory(answerDir); } - await File.WriteAllTextAsync(answerPath, executionResult.Output); + await File.WriteAllTextAsync(answerPath, executionResult.Output).ConfigureAwait(false); generatedCount++; - _logger.LogDebug("Generated answer {Index}/{Total}", i + 1, inputFilePaths.Count); + _logger.LogDebug("Generated answer {Index}/{Total}", i + 1, totalTests); } _logger.LogInformation("Generated {Count} answer files out of {Total} tests", - generatedCount, inputFilePaths.Count); + generatedCount, totalTests); return generatedCount > 0; } diff --git a/src/LiquidCode.Tester.Worker/Services/CSharpExecutionService.cs b/src/LiquidCode.Tester.Worker/Services/CSharpExecutionService.cs index 53fdc16..530abe4 100644 --- a/src/LiquidCode.Tester.Worker/Services/CSharpExecutionService.cs +++ b/src/LiquidCode.Tester.Worker/Services/CSharpExecutionService.cs @@ -1,108 +1,38 @@ -using System.Diagnostics; +using System.IO; +using LiquidCode.Tester.Worker.Services.Sandbox; namespace LiquidCode.Tester.Worker.Services; public class CSharpExecutionService : IExecutionService { private readonly ILogger _logger; + private readonly IExecutionSandbox _sandbox; - public CSharpExecutionService(ILogger logger) + public CSharpExecutionService(IExecutionSandbox sandbox, ILogger logger) { + _sandbox = sandbox; _logger = logger; } public async Task ExecuteAsync(string executablePath, string inputFilePath, int timeLimitMs, int memoryLimitMb) { - _logger.LogInformation("Executing C# executable {Executable} with input {Input}, time limit {TimeLimit}ms, memory limit {MemoryLimit}MB", - executablePath, inputFilePath, timeLimitMs, memoryLimitMb); + _logger.LogInformation("Executing C# executable {Executable} with input {Input}", executablePath, inputFilePath); - var result = new ExecutionResult(); - var stopwatch = Stopwatch.StartNew(); + var workingDirectory = Path.GetDirectoryName(executablePath) ?? Directory.GetCurrentDirectory(); + var executableFileName = Path.GetFileName(executablePath); - try + var request = new ExecutionSandboxRequest { - using var inputStream = File.OpenRead(inputFilePath); + Language = "csharp", + WorkingDirectory = workingDirectory, + Executable = $"./{executableFileName}", + TimeLimitMilliseconds = timeLimitMs, + MemoryLimitMegabytes = memoryLimitMb, + StandardInputFile = inputFilePath, + AllowNetwork = false + }; - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = executablePath, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - process.Start(); - - await inputStream.CopyToAsync(process.StandardInput.BaseStream); - process.StandardInput.Close(); - - var outputTask = process.StandardOutput.ReadToEndAsync(); - var errorTask = process.StandardError.ReadToEndAsync(); - - var completedInTime = await Task.Run(() => process.WaitForExit(timeLimitMs)); - - stopwatch.Stop(); - result.ExecutionTimeMs = stopwatch.ElapsedMilliseconds; - - if (!completedInTime) - { - try - { - process.Kill(entireProcessTree: true); - } - catch { } - - result.TimeLimitExceeded = true; - result.ErrorMessage = "Time limit exceeded"; - _logger.LogWarning("Execution exceeded time limit"); - return result; - } - - result.Output = await outputTask; - result.ErrorOutput = await errorTask; - result.ExitCode = process.ExitCode; - - if (process.ExitCode != 0) - { - result.RuntimeError = true; - result.ErrorMessage = $"Runtime error (exit code {process.ExitCode})"; - _logger.LogWarning("Runtime error with exit code {ExitCode}", process.ExitCode); - return result; - } - - try - { - result.MemoryUsedMb = process.PeakWorkingSet64 / (1024 * 1024); - if (result.MemoryUsedMb > memoryLimitMb) - { - result.MemoryLimitExceeded = true; - result.ErrorMessage = "Memory limit exceeded"; - _logger.LogWarning("Memory limit exceeded: {Used}MB > {Limit}MB", result.MemoryUsedMb, memoryLimitMb); - return result; - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Could not measure memory usage"); - } - - result.Success = true; - _logger.LogInformation("Execution completed successfully in {Time}ms", result.ExecutionTimeMs); - } - catch (Exception ex) - { - stopwatch.Stop(); - result.ExecutionTimeMs = stopwatch.ElapsedMilliseconds; - result.RuntimeError = true; - result.ErrorMessage = $"Execution error: {ex.Message}"; - _logger.LogError(ex, "Error during execution"); - } - - return result; + var sandboxResult = await _sandbox.RunAsync(request).ConfigureAwait(false); + return sandboxResult.ToExecutionResult(memoryLimitMb); } } diff --git a/src/LiquidCode.Tester.Worker/Services/CppExecutionService.cs b/src/LiquidCode.Tester.Worker/Services/CppExecutionService.cs index 411bfe5..f1e4b49 100644 --- a/src/LiquidCode.Tester.Worker/Services/CppExecutionService.cs +++ b/src/LiquidCode.Tester.Worker/Services/CppExecutionService.cs @@ -1,114 +1,191 @@ -using System.Diagnostics; +using System.IO; +using LiquidCode.Tester.Worker.Services.Sandbox; namespace LiquidCode.Tester.Worker.Services; public class CppExecutionService : IExecutionService { private readonly ILogger _logger; + private readonly IExecutionSandbox _sandbox; - public CppExecutionService(ILogger logger) + public CppExecutionService(IExecutionSandbox sandbox, ILogger logger) { + _sandbox = sandbox; _logger = logger; } public async Task ExecuteAsync(string executablePath, string inputFilePath, int timeLimitMs, int memoryLimitMb) { - _logger.LogInformation("Executing {Executable} with input {Input}, time limit {TimeLimit}ms, memory limit {MemoryLimit}MB", + _logger.LogInformation("Executing {Executable} with input {Input}, limits {TimeLimit}ms/{MemoryLimit}MB", executablePath, inputFilePath, timeLimitMs, memoryLimitMb); - var result = new ExecutionResult(); - var stopwatch = Stopwatch.StartNew(); + var workingDirectory = Path.GetDirectoryName(executablePath) ?? Directory.GetCurrentDirectory(); + var executableFileName = Path.GetFileName(executablePath); + + var request = new ExecutionSandboxRequest + { + Language = "cpp", + WorkingDirectory = workingDirectory, + Executable = $"./{executableFileName}", + TimeLimitMilliseconds = timeLimitMs, + MemoryLimitMegabytes = memoryLimitMb, + StandardInputFile = inputFilePath, + AllowNetwork = false + }; + + var sandboxResult = await _sandbox.RunAsync(request).ConfigureAwait(false); + return sandboxResult.ToExecutionResult(memoryLimitMb); + } + + public async Task ExecuteBatchAsync( + string executablePath, + string packageRootDirectory, + IReadOnlyList inputFilePaths, + IReadOnlyList outputFilePaths, + int perTestTimeLimitMs, + int memoryLimitMb, + CancellationToken cancellationToken = default) + { + if (inputFilePaths.Count == 0 || inputFilePaths.Count != outputFilePaths.Count) + { + _logger.LogWarning("Invalid batch execution request: inputs={Inputs}, outputs={Outputs}", inputFilePaths.Count, outputFilePaths.Count); + return null; + } + + var sandboxRoot = Path.GetFullPath(packageRootDirectory); + Directory.CreateDirectory(sandboxRoot); + + var sandboxWorkDir = Path.Combine(sandboxRoot, ".lc-sandbox"); + Directory.CreateDirectory(sandboxWorkDir); + + var scriptPath = Path.Combine(sandboxWorkDir, "run-batch.sh"); + var jobsPath = Path.Combine(sandboxWorkDir, "jobs.txt"); + + var executableRelative = NormalizeRelativePath(Path.GetRelativePath(sandboxRoot, executablePath)); + var scriptRelative = NormalizeRelativePath(Path.GetRelativePath(sandboxRoot, scriptPath)); + var jobsRelative = NormalizeRelativePath(Path.GetRelativePath(sandboxRoot, jobsPath)); + + if (executableRelative.StartsWith("../", StringComparison.Ordinal)) + { + _logger.LogWarning("Executable path {ExecutablePath} is outside of package root {PackageRoot}. Batch execution cancelled.", executablePath, sandboxRoot); + return null; + } + + var jobLines = new List(inputFilePaths.Count); + for (var i = 0; i < inputFilePaths.Count; i++) + { + var inputRelative = NormalizeRelativePath(Path.GetRelativePath(sandboxRoot, inputFilePaths[i])); + var outputRelative = NormalizeRelativePath(Path.GetRelativePath(sandboxRoot, outputFilePaths[i])); + + if (inputRelative.StartsWith("../", StringComparison.Ordinal) || outputRelative.StartsWith("../", StringComparison.Ordinal)) + { + _logger.LogWarning("Input or output path is outside of package root. Falling back to sequential execution."); + return null; + } + + jobLines.Add($"{inputRelative}|{outputRelative}"); + } + + await File.WriteAllLinesAsync(jobsPath, jobLines, cancellationToken).ConfigureAwait(false); + + var scriptContent = """ + #!/bin/sh + set -eu + + EXEC="$1" + JOBS_FILE="$2" + + rc=0 + + IFS='|' + while IFS='|' read -r input_path output_path || [ -n "$input_path" ]; do + if [ -z "$input_path" ]; then + continue + fi + + output_dir=$(dirname "$output_path") + mkdir -p "$output_dir" + + if ! "$EXEC" < "$input_path" > "$output_path"; then + status=$? + echo "FAILED $input_path (exit $status)" >&2 + + if [ "$rc" -eq 0 ]; then + rc=$status + fi + fi + done < "$JOBS_FILE" + + exit "$rc" + """; + + await File.WriteAllTextAsync(scriptPath, scriptContent, cancellationToken).ConfigureAwait(false); + + var request = new ExecutionSandboxRequest + { + Language = "cpp", + WorkingDirectory = sandboxRoot, + Executable = "/bin/sh", + TimeLimitMilliseconds = CalculateOverallTimeLimit(perTestTimeLimitMs, inputFilePaths.Count), + MemoryLimitMegabytes = Math.Max(memoryLimitMb, 1), + AllowNetwork = false + }; + + request.Arguments.Add(EnsureRelativeForShell(scriptRelative)); + request.Arguments.Add(EnsureRelativeForShell(executableRelative)); + request.Arguments.Add(EnsureRelativeForShell(jobsRelative)); + + _logger.LogInformation("Executing batch of {Count} tests inside sandbox (per-test limit {Limit} ms)", inputFilePaths.Count, perTestTimeLimitMs); try { - using var inputStream = File.OpenRead(inputFilePath); + return await _sandbox.RunAsync(request, cancellationToken).ConfigureAwait(false); + } + finally + { + TryDelete(scriptPath); + TryDelete(jobsPath); + } + } - var process = new Process + private static string NormalizeRelativePath(string path) => path.Replace('\\', '/'); + + private static string EnsureRelativeForShell(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return path; + } + + if (path.StartsWith("./", StringComparison.Ordinal) || path.StartsWith("/", StringComparison.Ordinal)) + { + return path; + } + + return $"./{path}"; + } + + private static int CalculateOverallTimeLimit(int perTestTimeLimitMs, int testCount) + { + var perTest = Math.Max(perTestTimeLimitMs, 1000); + var count = Math.Max(testCount, 1); + var buffer = Math.Max(perTest / 2, 1000); + var total = (long)perTest * count + buffer; + return total >= int.MaxValue ? int.MaxValue : (int)total; + } + + private void TryDelete(string path) + { + try + { + if (File.Exists(path)) { - StartInfo = new ProcessStartInfo - { - FileName = executablePath, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - process.Start(); - - // Copy input to stdin - await inputStream.CopyToAsync(process.StandardInput.BaseStream); - process.StandardInput.Close(); - - // Read output - var outputTask = process.StandardOutput.ReadToEndAsync(); - var errorTask = process.StandardError.ReadToEndAsync(); - - // Wait for completion with timeout - var completedInTime = await Task.Run(() => process.WaitForExit(timeLimitMs)); - - stopwatch.Stop(); - result.ExecutionTimeMs = stopwatch.ElapsedMilliseconds; - - if (!completedInTime) - { - // Time limit exceeded - try - { - process.Kill(entireProcessTree: true); - } - catch { } - - result.TimeLimitExceeded = true; - result.ErrorMessage = "Time limit exceeded"; - _logger.LogWarning("Execution exceeded time limit"); - return result; + File.Delete(path); } - - result.Output = await outputTask; - result.ErrorOutput = await errorTask; - result.ExitCode = process.ExitCode; - - // Check for runtime error - if (process.ExitCode != 0) - { - result.RuntimeError = true; - result.ErrorMessage = $"Runtime error (exit code {process.ExitCode})"; - _logger.LogWarning("Runtime error with exit code {ExitCode}", process.ExitCode); - return result; - } - - // Estimate memory usage (basic approach - in real scenario we'd use cgroups) - try - { - result.MemoryUsedMb = process.PeakWorkingSet64 / (1024 * 1024); - if (result.MemoryUsedMb > memoryLimitMb) - { - result.MemoryLimitExceeded = true; - result.ErrorMessage = "Memory limit exceeded"; - _logger.LogWarning("Memory limit exceeded: {Used}MB > {Limit}MB", result.MemoryUsedMb, memoryLimitMb); - return result; - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Could not measure memory usage"); - } - - result.Success = true; - _logger.LogInformation("Execution completed successfully in {Time}ms", result.ExecutionTimeMs); } catch (Exception ex) { - stopwatch.Stop(); - result.ExecutionTimeMs = stopwatch.ElapsedMilliseconds; - result.RuntimeError = true; - result.ErrorMessage = $"Execution error: {ex.Message}"; - _logger.LogError(ex, "Error during execution"); + _logger.LogDebug(ex, "Failed to delete temporary sandbox file {Path}", path); } - - return result; } } diff --git a/src/LiquidCode.Tester.Worker/Services/JavaExecutionService.cs b/src/LiquidCode.Tester.Worker/Services/JavaExecutionService.cs index 5c6786a..e6fceb8 100644 --- a/src/LiquidCode.Tester.Worker/Services/JavaExecutionService.cs +++ b/src/LiquidCode.Tester.Worker/Services/JavaExecutionService.cs @@ -1,111 +1,37 @@ -using System.Diagnostics; +using System.IO; +using LiquidCode.Tester.Worker.Services.Sandbox; namespace LiquidCode.Tester.Worker.Services; public class JavaExecutionService : IExecutionService { private readonly ILogger _logger; + private readonly IExecutionSandbox _sandbox; - public JavaExecutionService(ILogger logger) + public JavaExecutionService(IExecutionSandbox sandbox, ILogger logger) { + _sandbox = sandbox; _logger = logger; } public async Task ExecuteAsync(string executablePath, string inputFilePath, int timeLimitMs, int memoryLimitMb) { - var workingDirectory = Path.GetDirectoryName(executablePath)!; - _logger.LogInformation("Executing Java class in {WorkingDirectory} with input {Input}, time limit {TimeLimit}ms, memory limit {MemoryLimit}MB", - workingDirectory, inputFilePath, timeLimitMs, memoryLimitMb); + var workingDirectory = Path.GetDirectoryName(executablePath) ?? Directory.GetCurrentDirectory(); + _logger.LogInformation("Executing Java class from {WorkingDirectory} with input {Input}", workingDirectory, inputFilePath); - var result = new ExecutionResult(); - var stopwatch = Stopwatch.StartNew(); - - try + var request = new ExecutionSandboxRequest { - using var inputStream = File.OpenRead(inputFilePath); + Language = "java", + WorkingDirectory = workingDirectory, + Executable = "java", + Arguments = { "-cp", ".", "Solution" }, + TimeLimitMilliseconds = timeLimitMs, + MemoryLimitMegabytes = memoryLimitMb, + StandardInputFile = inputFilePath, + AllowNetwork = false + }; - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = "java", - Arguments = "-cp . Solution", - WorkingDirectory = workingDirectory, - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - process.Start(); - - await inputStream.CopyToAsync(process.StandardInput.BaseStream); - process.StandardInput.Close(); - - var outputTask = process.StandardOutput.ReadToEndAsync(); - var errorTask = process.StandardError.ReadToEndAsync(); - - var completedInTime = await Task.Run(() => process.WaitForExit(timeLimitMs)); - - stopwatch.Stop(); - result.ExecutionTimeMs = stopwatch.ElapsedMilliseconds; - - if (!completedInTime) - { - try - { - process.Kill(entireProcessTree: true); - } - catch { } - - result.TimeLimitExceeded = true; - result.ErrorMessage = "Time limit exceeded"; - _logger.LogWarning("Execution exceeded time limit"); - return result; - } - - result.Output = await outputTask; - result.ErrorOutput = await errorTask; - result.ExitCode = process.ExitCode; - - if (process.ExitCode != 0) - { - result.RuntimeError = true; - result.ErrorMessage = $"Runtime error (exit code {process.ExitCode})"; - _logger.LogWarning("Runtime error with exit code {ExitCode}", process.ExitCode); - return result; - } - - try - { - result.MemoryUsedMb = process.PeakWorkingSet64 / (1024 * 1024); - if (result.MemoryUsedMb > memoryLimitMb) - { - result.MemoryLimitExceeded = true; - result.ErrorMessage = "Memory limit exceeded"; - _logger.LogWarning("Memory limit exceeded: {Used}MB > {Limit}MB", result.MemoryUsedMb, memoryLimitMb); - return result; - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Could not measure memory usage"); - } - - result.Success = true; - _logger.LogInformation("Execution completed successfully in {Time}ms", result.ExecutionTimeMs); - } - catch (Exception ex) - { - stopwatch.Stop(); - result.ExecutionTimeMs = stopwatch.ElapsedMilliseconds; - result.RuntimeError = true; - result.ErrorMessage = $"Execution error: {ex.Message}"; - _logger.LogError(ex, "Error during execution"); - } - - return result; + var sandboxResult = await _sandbox.RunAsync(request).ConfigureAwait(false); + return sandboxResult.ToExecutionResult(memoryLimitMb); } } diff --git a/src/LiquidCode.Tester.Worker/Services/KotlinExecutionService.cs b/src/LiquidCode.Tester.Worker/Services/KotlinExecutionService.cs index 5571b01..5040be7 100644 --- a/src/LiquidCode.Tester.Worker/Services/KotlinExecutionService.cs +++ b/src/LiquidCode.Tester.Worker/Services/KotlinExecutionService.cs @@ -1,110 +1,39 @@ -using System.Diagnostics; +using System.IO; +using LiquidCode.Tester.Worker.Services.Sandbox; namespace LiquidCode.Tester.Worker.Services; public class KotlinExecutionService : IExecutionService { private readonly ILogger _logger; + private readonly IExecutionSandbox _sandbox; - public KotlinExecutionService(ILogger logger) + public KotlinExecutionService(IExecutionSandbox sandbox, ILogger logger) { + _sandbox = sandbox; _logger = logger; } public async Task ExecuteAsync(string executablePath, string inputFilePath, int timeLimitMs, int memoryLimitMb) { - _logger.LogInformation("Executing Kotlin JAR {Executable} with input {Input}, time limit {TimeLimit}ms, memory limit {MemoryLimit}MB", - executablePath, inputFilePath, timeLimitMs, memoryLimitMb); + _logger.LogInformation("Executing Kotlin jar {Executable} with input {Input}", executablePath, inputFilePath); - var result = new ExecutionResult(); - var stopwatch = Stopwatch.StartNew(); + var workingDirectory = Path.GetDirectoryName(executablePath) ?? Directory.GetCurrentDirectory(); + var executableFileName = Path.GetFileName(executablePath); - try + var request = new ExecutionSandboxRequest { - using var inputStream = File.OpenRead(inputFilePath); + Language = "kotlin", + WorkingDirectory = workingDirectory, + Executable = "java", + Arguments = { "-jar", executableFileName }, + TimeLimitMilliseconds = timeLimitMs, + MemoryLimitMegabytes = memoryLimitMb, + StandardInputFile = inputFilePath, + AllowNetwork = false + }; - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = "java", - Arguments = $"-jar \"{executablePath}\"", - WorkingDirectory = Path.GetDirectoryName(executablePath), - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - process.Start(); - - await inputStream.CopyToAsync(process.StandardInput.BaseStream); - process.StandardInput.Close(); - - var outputTask = process.StandardOutput.ReadToEndAsync(); - var errorTask = process.StandardError.ReadToEndAsync(); - - var completedInTime = await Task.Run(() => process.WaitForExit(timeLimitMs)); - - stopwatch.Stop(); - result.ExecutionTimeMs = stopwatch.ElapsedMilliseconds; - - if (!completedInTime) - { - try - { - process.Kill(entireProcessTree: true); - } - catch { } - - result.TimeLimitExceeded = true; - result.ErrorMessage = "Time limit exceeded"; - _logger.LogWarning("Execution exceeded time limit"); - return result; - } - - result.Output = await outputTask; - result.ErrorOutput = await errorTask; - result.ExitCode = process.ExitCode; - - if (process.ExitCode != 0) - { - result.RuntimeError = true; - result.ErrorMessage = $"Runtime error (exit code {process.ExitCode})"; - _logger.LogWarning("Runtime error with exit code {ExitCode}", process.ExitCode); - return result; - } - - try - { - result.MemoryUsedMb = process.PeakWorkingSet64 / (1024 * 1024); - if (result.MemoryUsedMb > memoryLimitMb) - { - result.MemoryLimitExceeded = true; - result.ErrorMessage = "Memory limit exceeded"; - _logger.LogWarning("Memory limit exceeded: {Used}MB > {Limit}MB", result.MemoryUsedMb, memoryLimitMb); - return result; - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Could not measure memory usage"); - } - - result.Success = true; - _logger.LogInformation("Execution completed successfully in {Time}ms", result.ExecutionTimeMs); - } - catch (Exception ex) - { - stopwatch.Stop(); - result.ExecutionTimeMs = stopwatch.ElapsedMilliseconds; - result.RuntimeError = true; - result.ErrorMessage = $"Execution error: {ex.Message}"; - _logger.LogError(ex, "Error during execution"); - } - - return result; + var sandboxResult = await _sandbox.RunAsync(request).ConfigureAwait(false); + return sandboxResult.ToExecutionResult(memoryLimitMb); } } diff --git a/src/LiquidCode.Tester.Worker/Services/PackageParserService.cs b/src/LiquidCode.Tester.Worker/Services/PackageParserService.cs index b887c27..eb3cf00 100644 --- a/src/LiquidCode.Tester.Worker/Services/PackageParserService.cs +++ b/src/LiquidCode.Tester.Worker/Services/PackageParserService.cs @@ -89,7 +89,7 @@ public class PackageParserService : IPackageParserService package.CheckerPath = await CompileCheckerForPackageAsync(descriptor, packageRoot, buildDirectory) ?? await FindAndCompileCheckerAsync(packageRoot); - var validatorPath = await CompileValidatorAsync(descriptor, packageRoot, buildDirectory, compiledExecutables); + var validatorPath = await CompileValidatorAsync(descriptor, packageRoot, buildDirectory, compiledExecutables); await GenerateAndValidateTestsAsync(descriptor, packageRoot, compiledExecutables, validatorPath); @@ -103,41 +103,53 @@ public class PackageParserService : IPackageParserService return await ParseLegacyPackage(packageRoot, extractionRoot); } - var inputs = new List<(int index, string inputPath)>(); - var answers = new Dictionary(); + var testDefinitionByIndex = descriptor.Tests + .GroupBy(t => t.Index) + .ToDictionary(g => g.Key, g => g.First()); + + var testDescriptors = new List(testIndices.Count); foreach (var testIndex in testIndices) { var inputRelative = FormatPolygonPattern(descriptor.InputPathPattern, testIndex); var inputFullPath = Path.Combine(packageRoot, NormalizeRelativePath(inputRelative)); - - if (!File.Exists(inputFullPath)) - { - _logger.LogWarning("Input file not found for test {Index}: {RelativePath}", testIndex, inputRelative); - continue; - } - var answerRelative = FormatPolygonPattern(descriptor.AnswerPathPattern, testIndex); var answerFullPath = Path.Combine(packageRoot, NormalizeRelativePath(answerRelative)); - inputs.Add((testIndex, inputFullPath)); - answers[testIndex] = answerFullPath; + testDescriptors.Add(new TestFileDescriptor( + testIndex, + inputRelative, + inputFullPath, + answerRelative, + answerFullPath)); } + var missingInputIndices = new HashSet(); var missingAnswerInputs = new List(); var missingAnswerPaths = new List(); - foreach (var (index, inputPath) in inputs) + foreach (var testDescriptor in testDescriptors) { - if (!answers.TryGetValue(index, out var answerPath)) + if (!File.Exists(testDescriptor.InputFullPath)) { + var method = testDefinitionByIndex.TryGetValue(testDescriptor.Index, out var definition) + ? definition.Method.ToString().ToLowerInvariant() + : "unknown"; + + _logger.LogWarning( + "Input file not found for test {Index} ({Method}): {RelativePath}", + testDescriptor.Index, + method, + testDescriptor.InputRelativePath); + + missingInputIndices.Add(testDescriptor.Index); continue; } - if (!File.Exists(answerPath)) + if (!File.Exists(testDescriptor.AnswerFullPath)) { - missingAnswerInputs.Add(inputPath); - missingAnswerPaths.Add(answerPath); + missingAnswerInputs.Add(testDescriptor.InputFullPath); + missingAnswerPaths.Add(testDescriptor.AnswerFullPath); } } @@ -161,24 +173,35 @@ public class PackageParserService : IPackageParserService } } - foreach (var (index, inputPath) in inputs.OrderBy(item => item.index)) + foreach (var testDescriptor in testDescriptors.OrderBy(t => t.Index)) { - if (!answers.TryGetValue(index, out var answerPath)) + if (!File.Exists(testDescriptor.InputFullPath)) { + if (!missingInputIndices.Contains(testDescriptor.Index)) + { + _logger.LogWarning( + "Input file not found for test {Index}: {RelativePath}", + testDescriptor.Index, + testDescriptor.InputRelativePath); + } + continue; } - if (!File.Exists(answerPath)) + if (!File.Exists(testDescriptor.AnswerFullPath)) { - _logger.LogWarning("Answer file not found for test {Index}: {AnswerPath}", index, answerPath); + _logger.LogWarning( + "Answer file not found for test {Index}: {RelativePath}", + testDescriptor.Index, + testDescriptor.AnswerRelativePath); continue; } package.TestCases.Add(new TestCase { - Number = index, - InputFilePath = inputPath, - OutputFilePath = answerPath, + Number = testDescriptor.Index, + InputFilePath = testDescriptor.InputFullPath, + OutputFilePath = testDescriptor.AnswerFullPath, TimeLimit = descriptor.TimeLimitMs, MemoryLimit = descriptor.MemoryLimitMb }); @@ -787,4 +810,11 @@ public class PackageParserService : IPackageParserService _logger.LogWarning("No checker found in package"); return null; } + + private sealed record TestFileDescriptor( + int Index, + string InputRelativePath, + string InputFullPath, + string AnswerRelativePath, + string AnswerFullPath); } diff --git a/src/LiquidCode.Tester.Worker/Services/PythonExecutionService.cs b/src/LiquidCode.Tester.Worker/Services/PythonExecutionService.cs index ed5dbd0..43f7cb4 100644 --- a/src/LiquidCode.Tester.Worker/Services/PythonExecutionService.cs +++ b/src/LiquidCode.Tester.Worker/Services/PythonExecutionService.cs @@ -1,4 +1,5 @@ -using System.Diagnostics; +using System.IO; +using LiquidCode.Tester.Worker.Services.Sandbox; namespace LiquidCode.Tester.Worker.Services; @@ -6,109 +7,36 @@ public class PythonExecutionService : IExecutionService { private readonly ILogger _logger; private readonly IConfiguration _configuration; + private readonly IExecutionSandbox _sandbox; - public PythonExecutionService(ILogger logger, IConfiguration configuration) + public PythonExecutionService(ILogger logger, IConfiguration configuration, IExecutionSandbox sandbox) { _logger = logger; _configuration = configuration; + _sandbox = sandbox; } public async Task ExecuteAsync(string executablePath, string inputFilePath, int timeLimitMs, int memoryLimitMb) { - _logger.LogInformation("Executing Python script {Executable} with input {Input}, time limit {TimeLimit}ms, memory limit {MemoryLimit}MB", - executablePath, inputFilePath, timeLimitMs, memoryLimitMb); + _logger.LogInformation("Executing Python script {Executable} with input {Input}", executablePath, inputFilePath); - var result = new ExecutionResult(); - var stopwatch = Stopwatch.StartNew(); + var pythonExecutable = _configuration["Python:Executable"] ?? "python3"; + var workingDirectory = Path.GetDirectoryName(executablePath) ?? Directory.GetCurrentDirectory(); + var scriptFileName = Path.GetFileName(executablePath); - try + var request = new ExecutionSandboxRequest { - using var inputStream = File.OpenRead(inputFilePath); + Language = "python", + WorkingDirectory = workingDirectory, + Executable = pythonExecutable, + Arguments = { scriptFileName }, + TimeLimitMilliseconds = timeLimitMs, + MemoryLimitMegabytes = memoryLimitMb, + StandardInputFile = inputFilePath, + AllowNetwork = false + }; - var pythonExecutable = _configuration["Python:Executable"] ?? "python3"; - - var process = new Process - { - StartInfo = new ProcessStartInfo - { - FileName = pythonExecutable, - Arguments = $"\"{executablePath}\"", - WorkingDirectory = Path.GetDirectoryName(executablePath), - RedirectStandardInput = true, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - } - }; - - process.Start(); - - await inputStream.CopyToAsync(process.StandardInput.BaseStream); - process.StandardInput.Close(); - - var outputTask = process.StandardOutput.ReadToEndAsync(); - var errorTask = process.StandardError.ReadToEndAsync(); - - var completedInTime = await Task.Run(() => process.WaitForExit(timeLimitMs)); - - stopwatch.Stop(); - result.ExecutionTimeMs = stopwatch.ElapsedMilliseconds; - - if (!completedInTime) - { - try - { - process.Kill(entireProcessTree: true); - } - catch { } - - result.TimeLimitExceeded = true; - result.ErrorMessage = "Time limit exceeded"; - _logger.LogWarning("Execution exceeded time limit"); - return result; - } - - result.Output = await outputTask; - result.ErrorOutput = await errorTask; - result.ExitCode = process.ExitCode; - - if (process.ExitCode != 0) - { - result.RuntimeError = true; - result.ErrorMessage = $"Runtime error (exit code {process.ExitCode})"; - _logger.LogWarning("Runtime error with exit code {ExitCode}", process.ExitCode); - return result; - } - - try - { - result.MemoryUsedMb = process.PeakWorkingSet64 / (1024 * 1024); - if (result.MemoryUsedMb > memoryLimitMb) - { - result.MemoryLimitExceeded = true; - result.ErrorMessage = "Memory limit exceeded"; - _logger.LogWarning("Memory limit exceeded: {Used}MB > {Limit}MB", result.MemoryUsedMb, memoryLimitMb); - return result; - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Could not measure memory usage"); - } - - result.Success = true; - _logger.LogInformation("Execution completed successfully in {Time}ms", result.ExecutionTimeMs); - } - catch (Exception ex) - { - stopwatch.Stop(); - result.ExecutionTimeMs = stopwatch.ElapsedMilliseconds; - result.RuntimeError = true; - result.ErrorMessage = $"Execution error: {ex.Message}"; - _logger.LogError(ex, "Error during execution"); - } - - return result; + var sandboxResult = await _sandbox.RunAsync(request).ConfigureAwait(false); + return sandboxResult.ToExecutionResult(memoryLimitMb); } } diff --git a/src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionResultMapper.cs b/src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionResultMapper.cs new file mode 100644 index 0000000..d6e8819 --- /dev/null +++ b/src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionResultMapper.cs @@ -0,0 +1,43 @@ +namespace LiquidCode.Tester.Worker.Services.Sandbox; + +public static class ExecutionResultMapper +{ + public static ExecutionResult ToExecutionResult(this ExecutionSandboxResult sandboxResult, int memoryLimitMb) + { + var executionResult = new ExecutionResult + { + Success = sandboxResult.Success, + Output = sandboxResult.StandardOutput, + ErrorOutput = sandboxResult.StandardError, + ExitCode = sandboxResult.ExitCode, + ExecutionTimeMs = sandboxResult.ExecutionTimeMilliseconds, + MemoryUsedMb = sandboxResult.MemoryUsageBytes.HasValue ? sandboxResult.MemoryUsageBytes.Value / (1024 * 1024) : 0, + TimeLimitExceeded = sandboxResult.TimedOut, + MemoryLimitExceeded = sandboxResult.MemoryLimitExceeded, + ErrorMessage = sandboxResult.ErrorMessage ?? string.Empty, + RuntimeError = !sandboxResult.Success && !sandboxResult.TimedOut && !sandboxResult.MemoryLimitExceeded + }; + + if (sandboxResult.MemoryUsageBytes.HasValue) + { + executionResult.MemoryUsedMb = sandboxResult.MemoryUsageBytes.Value / (1024 * 1024); + } + + if (sandboxResult.MemoryLimitExceeded && string.IsNullOrEmpty(executionResult.ErrorMessage)) + { + executionResult.ErrorMessage = $"Memory limit exceeded ({memoryLimitMb} MB)"; + } + + if (sandboxResult.TimedOut && string.IsNullOrEmpty(executionResult.ErrorMessage)) + { + executionResult.ErrorMessage = "Time limit exceeded"; + } + + if (!sandboxResult.Success && string.IsNullOrEmpty(executionResult.ErrorMessage)) + { + executionResult.ErrorMessage = "Runtime error"; + } + + return executionResult; + } +} diff --git a/src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionSandbox.cs b/src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionSandbox.cs new file mode 100644 index 0000000..bcef659 --- /dev/null +++ b/src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionSandbox.cs @@ -0,0 +1,435 @@ +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace LiquidCode.Tester.Worker.Services.Sandbox; + +public class ExecutionSandbox : IExecutionSandbox +{ + private readonly SandboxOptions _options; + private readonly ILogger _logger; + + public ExecutionSandbox(IOptions options, ILogger logger) + { + _options = options.Value; + _logger = logger; + } + + public async Task RunAsync(ExecutionSandboxRequest request, CancellationToken cancellationToken = default) + { + if (!_options.Enabled) + { + return await RunLocallyAsync(request, cancellationToken).ConfigureAwait(false); + } + + if (!_options.Languages.TryGetValue(request.Language, out var languageOptions) || string.IsNullOrWhiteSpace(languageOptions.ExecutionImage)) + { + _logger.LogWarning("Sandbox enabled but no execution image configured for language {Language}. Falling back to local execution.", request.Language); + return await RunLocallyAsync(request, cancellationToken).ConfigureAwait(false); + } + + try + { + return await RunInDockerAsync(request, languageOptions, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to execute inside Docker sandbox for language {Language}. Falling back to local execution.", request.Language); + return await RunLocallyAsync(request, cancellationToken).ConfigureAwait(false); + } + } + + private async Task RunLocallyAsync(ExecutionSandboxRequest request, CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + + var workingDirectory = string.IsNullOrWhiteSpace(request.WorkingDirectory) + ? Directory.GetCurrentDirectory() + : request.WorkingDirectory; + + var resolvedExecutablePath = Path.IsPathRooted(request.Executable) + ? request.Executable + : Path.GetFullPath(Path.Combine(workingDirectory, request.Executable)); + + if (!Directory.Exists(workingDirectory)) + { + _logger.LogWarning("Working directory not found for execution: {WorkingDirectory}", workingDirectory); + } + + if (!File.Exists(resolvedExecutablePath)) + { + _logger.LogWarning( + "Executable not found at resolved path {ResolvedPath} (WorkingDirectory: {WorkingDirectory}, Executable: {Executable})", + resolvedExecutablePath, + workingDirectory, + request.Executable); + } + + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = resolvedExecutablePath, + WorkingDirectory = workingDirectory, + RedirectStandardInput = request.StandardInputFile is not null, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }, + EnableRaisingEvents = true + }; + + foreach (var argument in request.Arguments) + { + process.StartInfo.ArgumentList.Add(argument); + } + + try + { + process.Start(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to start local execution process"); + return new ExecutionSandboxResult + { + Success = false, + ExitCode = -1, + ErrorMessage = ex.Message, + StandardError = ex.ToString(), + ExecutionTimeMilliseconds = stopwatch.ElapsedMilliseconds + }; + } + + var stdoutTask = process.StandardOutput.ReadToEndAsync(); + var stderrTask = process.StandardError.ReadToEndAsync(); + + Task? stdinTask = null; + if (request.StandardInputFile is not null) + { + stdinTask = Task.Run(async () => + { + try + { + await using var inputStream = File.OpenRead(request.StandardInputFile); + await inputStream.CopyToAsync(process.StandardInput.BaseStream, cancellationToken).ConfigureAwait(false); + await process.StandardInput.FlushAsync().ConfigureAwait(false); + process.StandardInput.Close(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to pipe input file {InputFile} to process", request.StandardInputFile); + } + }, cancellationToken); + } + + var timeout = TimeSpan.FromMilliseconds(request.TimeLimitMilliseconds); + var waitForExitTask = process.WaitForExitAsync(cancellationToken); + var timeoutTask = Task.Delay(timeout); + var completedTask = await Task.WhenAny(waitForExitTask, timeoutTask).ConfigureAwait(false); + + if (completedTask == timeoutTask) + { + TryKillProcessTree(process); + stopwatch.Stop(); + var stdout = await stdoutTask.ConfigureAwait(false); + var stderr = await stderrTask.ConfigureAwait(false); + + return new ExecutionSandboxResult + { + Success = false, + TimedOut = true, + StandardOutput = stdout, + StandardError = stderr, + ExecutionTimeMilliseconds = stopwatch.ElapsedMilliseconds, + ErrorMessage = "Time limit exceeded" + }; + } + + if (stdinTask is not null) + { + await stdinTask.ConfigureAwait(false); + } + + await waitForExitTask.ConfigureAwait(false); + stopwatch.Stop(); + + var output = await stdoutTask.ConfigureAwait(false); + var errorOutput = await stderrTask.ConfigureAwait(false); + var exitCode = process.ExitCode; + + var peakMemoryBytes = GetPeakMemory(process); + var memoryLimitBytes = request.MemoryLimitMegabytes > 0 ? request.MemoryLimitMegabytes * 1024L * 1024L : (long?)null; + var memoryExceeded = memoryLimitBytes.HasValue && peakMemoryBytes.HasValue && peakMemoryBytes.Value > memoryLimitBytes.Value; + + return new ExecutionSandboxResult + { + Success = exitCode == 0 && !memoryExceeded, + ExitCode = exitCode, + StandardOutput = output, + StandardError = errorOutput, + MemoryUsageBytes = peakMemoryBytes, + MemoryLimitExceeded = memoryExceeded, + ExecutionTimeMilliseconds = stopwatch.ElapsedMilliseconds, + ErrorMessage = exitCode == 0 ? null : $"Runtime error (exit code {exitCode})" + }; + } + + private static void TryKillProcessTree(Process process) + { + try + { + process.Kill(entireProcessTree: true); + } + catch + { + } + } + + private static long? GetPeakMemory(Process process) + { + try + { + return process.PeakWorkingSet64; + } + catch + { + return null; + } + } + + private async Task RunInDockerAsync(ExecutionSandboxRequest request, SandboxLanguageOptions languageOptions, CancellationToken cancellationToken) + { + var timeout = TimeSpan.FromMilliseconds(request.TimeLimitMilliseconds); + var memoryLimit = languageOptions.MemoryLimitMb ?? _options.DefaultMemoryLimitMb; + var cpuLimit = languageOptions.CpuLimit ?? _options.DefaultCpuLimit; + var pidsLimit = languageOptions.PidsLimit ?? _options.DefaultPidsLimit; + + var containerName = $"lc-sbx-{Guid.NewGuid():N}"; + var workingDirectory = Path.GetFullPath(request.WorkingDirectory); + + if (!await EnsureDockerImageAvailableAsync(languageOptions.ExecutionImage!, cancellationToken).ConfigureAwait(false)) + { + _logger.LogWarning("Docker image {Image} is not available locally. Falling back to local execution.", languageOptions.ExecutionImage); + return await RunLocallyAsync(request, cancellationToken).ConfigureAwait(false); + } + + var processStartInfo = new ProcessStartInfo + { + FileName = _options.DockerExecutable, + RedirectStandardInput = request.StandardInputFile is not null, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + processStartInfo.ArgumentList.Add("run"); + processStartInfo.ArgumentList.Add("--rm"); + processStartInfo.ArgumentList.Add("--name"); + processStartInfo.ArgumentList.Add(containerName); + + if (!request.AllowNetwork) + { + processStartInfo.ArgumentList.Add("--network"); + processStartInfo.ArgumentList.Add("none"); + } + + processStartInfo.ArgumentList.Add("--cpus"); + processStartInfo.ArgumentList.Add(cpuLimit.ToString(CultureInfo.InvariantCulture)); + + processStartInfo.ArgumentList.Add($"--memory={memoryLimit}m"); + processStartInfo.ArgumentList.Add("--pids-limit"); + processStartInfo.ArgumentList.Add(pidsLimit.ToString(CultureInfo.InvariantCulture)); + + processStartInfo.ArgumentList.Add("--security-opt"); + processStartInfo.ArgumentList.Add("no-new-privileges"); + processStartInfo.ArgumentList.Add("--cap-drop=ALL"); + processStartInfo.ArgumentList.Add("--read-only"); + processStartInfo.ArgumentList.Add("--tmpfs"); + processStartInfo.ArgumentList.Add("/tmp:rw,noexec,nosuid,size=64m"); + + processStartInfo.ArgumentList.Add("-v"); + processStartInfo.ArgumentList.Add($"{workingDirectory}:/workspace:rw"); + processStartInfo.ArgumentList.Add("-w"); + processStartInfo.ArgumentList.Add("/workspace"); + + processStartInfo.ArgumentList.Add(languageOptions.ExecutionImage!); + processStartInfo.ArgumentList.Add(request.Executable); + foreach (var argument in request.Arguments) + { + processStartInfo.ArgumentList.Add(argument); + } + + var process = new Process { StartInfo = processStartInfo, EnableRaisingEvents = true }; + var stopwatch = Stopwatch.StartNew(); + + try + { + process.Start(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to start docker process. Falling back to local execution."); + return await RunLocallyAsync(request, cancellationToken).ConfigureAwait(false); + } + + Task? stdinTask = null; + if (request.StandardInputFile is not null) + { + stdinTask = Task.Run(async () => + { + try + { + await using var inputStream = File.OpenRead(request.StandardInputFile); + await inputStream.CopyToAsync(process.StandardInput.BaseStream, cancellationToken).ConfigureAwait(false); + await process.StandardInput.FlushAsync().ConfigureAwait(false); + process.StandardInput.Close(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to stream stdin to docker sandbox"); + } + }, cancellationToken); + } + + var stdoutTask = process.StandardOutput.ReadToEndAsync(); + var stderrTask = process.StandardError.ReadToEndAsync(); + + var waitForExitTask = process.WaitForExitAsync(cancellationToken); + var timeoutTask = Task.Delay(timeout); + var completedTask = await Task.WhenAny(waitForExitTask, timeoutTask).ConfigureAwait(false); + if (completedTask == timeoutTask) + { + await KillContainerAsync(containerName).ConfigureAwait(false); + stopwatch.Stop(); + var stdout = await stdoutTask.ConfigureAwait(false); + var stderr = await stderrTask.ConfigureAwait(false); + + return new ExecutionSandboxResult + { + Success = false, + TimedOut = true, + StandardOutput = stdout, + StandardError = stderr, + ExecutionTimeMilliseconds = stopwatch.ElapsedMilliseconds, + ErrorMessage = "Time limit exceeded", + UsedSandbox = true + }; + } + + if (stdinTask is not null) + { + await stdinTask.ConfigureAwait(false); + } + + await waitForExitTask.ConfigureAwait(false); + stopwatch.Stop(); + + var output = await stdoutTask.ConfigureAwait(false); + var errorOutput = await stderrTask.ConfigureAwait(false); + var exitCode = process.ExitCode; + + var memoryExceeded = exitCode == 137; + + return new ExecutionSandboxResult + { + Success = exitCode == 0 && !memoryExceeded, + ExitCode = exitCode, + StandardOutput = output, + StandardError = errorOutput, + MemoryLimitExceeded = memoryExceeded, + ExecutionTimeMilliseconds = stopwatch.ElapsedMilliseconds, + ErrorMessage = exitCode == 0 ? null : (memoryExceeded ? "Memory limit exceeded" : $"Runtime error (exit code {exitCode})"), + UsedSandbox = true + }; + } + + private async Task EnsureDockerImageAvailableAsync(string executionImage, CancellationToken cancellationToken) + { + try + { + using var inspectProcess = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = _options.DockerExecutable, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + } + }; + + inspectProcess.StartInfo.ArgumentList.Add("image"); + inspectProcess.StartInfo.ArgumentList.Add("inspect"); + inspectProcess.StartInfo.ArgumentList.Add("--format"); + inspectProcess.StartInfo.ArgumentList.Add("{{.Id}}"); + inspectProcess.StartInfo.ArgumentList.Add(executionImage); + + inspectProcess.Start(); + + var stdOutTask = inspectProcess.StandardOutput.ReadToEndAsync(); + var stdErrTask = inspectProcess.StandardError.ReadToEndAsync(); + + await inspectProcess.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + + var stdOut = (await stdOutTask.ConfigureAwait(false)).Trim(); + var stdErr = (await stdErrTask.ConfigureAwait(false)).Trim(); + + if (inspectProcess.ExitCode == 0) + { + if (!string.IsNullOrEmpty(stdOut)) + { + _logger.LogDebug("Docker image {Image} available: {ImageId}", executionImage, stdOut); + } + return true; + } + + if (!string.IsNullOrEmpty(stdErr)) + { + _logger.LogWarning("Docker image inspect failed for {Image}: {Error}", executionImage, stdErr); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to inspect docker image {Image}", executionImage); + } + + return false; + } + + private async Task KillContainerAsync(string containerName) + { + try + { + using var killProcess = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = _options.DockerExecutable, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + } + }; + + killProcess.StartInfo.ArgumentList.Add("kill"); + killProcess.StartInfo.ArgumentList.Add(containerName); + + killProcess.Start(); + await killProcess.WaitForExitAsync(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to kill docker container {ContainerName}", containerName); + } + } +} diff --git a/src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionSandboxModels.cs b/src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionSandboxModels.cs new file mode 100644 index 0000000..728ac05 --- /dev/null +++ b/src/LiquidCode.Tester.Worker/Services/Sandbox/ExecutionSandboxModels.cs @@ -0,0 +1,27 @@ +namespace LiquidCode.Tester.Worker.Services.Sandbox; + +public class ExecutionSandboxRequest +{ + public required string Language { get; init; } + public required string WorkingDirectory { get; init; } + public required string Executable { get; init; } + public IList Arguments { get; init; } = new List(); + public string? StandardInputFile { get; init; } + public int TimeLimitMilliseconds { get; init; } + public int MemoryLimitMegabytes { get; init; } + public bool AllowNetwork { get; init; } +} + +public class ExecutionSandboxResult +{ + public bool Success { get; init; } + public bool TimedOut { get; init; } + public bool MemoryLimitExceeded { get; init; } + public int ExitCode { get; init; } + public string StandardOutput { get; init; } = string.Empty; + public string StandardError { get; init; } = string.Empty; + public long? MemoryUsageBytes { get; init; } + public long ExecutionTimeMilliseconds { get; init; } + public string? ErrorMessage { get; init; } + public bool UsedSandbox { get; init; } +} diff --git a/src/LiquidCode.Tester.Worker/Services/Sandbox/IExecutionSandbox.cs b/src/LiquidCode.Tester.Worker/Services/Sandbox/IExecutionSandbox.cs new file mode 100644 index 0000000..f67a81e --- /dev/null +++ b/src/LiquidCode.Tester.Worker/Services/Sandbox/IExecutionSandbox.cs @@ -0,0 +1,9 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace LiquidCode.Tester.Worker.Services.Sandbox; + +public interface IExecutionSandbox +{ + Task RunAsync(ExecutionSandboxRequest request, CancellationToken cancellationToken = default); +} diff --git a/src/LiquidCode.Tester.Worker/Services/Sandbox/SandboxOptions.cs b/src/LiquidCode.Tester.Worker/Services/Sandbox/SandboxOptions.cs new file mode 100644 index 0000000..5870aa5 --- /dev/null +++ b/src/LiquidCode.Tester.Worker/Services/Sandbox/SandboxOptions.cs @@ -0,0 +1,21 @@ +namespace LiquidCode.Tester.Worker.Services.Sandbox; + +public class SandboxOptions +{ + public bool Enabled { get; set; } + public string DockerExecutable { get; set; } = "docker"; + public double DefaultCpuLimit { get; set; } = 1.0; + public int DefaultMemoryLimitMb { get; set; } = 512; + public int DefaultPidsLimit { get; set; } = 128; + public IDictionary Languages { get; set; } = + new Dictionary(StringComparer.OrdinalIgnoreCase); +} + +public class SandboxLanguageOptions +{ + public string? ExecutionImage { get; set; } + public string? CompilationImage { get; set; } + public double? CpuLimit { get; set; } + public int? MemoryLimitMb { get; set; } + public int? PidsLimit { get; set; } +} diff --git a/src/LiquidCode.Tester.Worker/appsettings.Development.json b/src/LiquidCode.Tester.Worker/appsettings.Development.json index 34f00ef..1004439 100644 --- a/src/LiquidCode.Tester.Worker/appsettings.Development.json +++ b/src/LiquidCode.Tester.Worker/appsettings.Development.json @@ -4,5 +4,8 @@ "Default": "Debug", "Microsoft.AspNetCore": "Information" } + }, + "Sandbox": { + "Enabled": true } } diff --git a/src/LiquidCode.Tester.Worker/appsettings.json b/src/LiquidCode.Tester.Worker/appsettings.json index 7ec48f4..ce76de1 100644 --- a/src/LiquidCode.Tester.Worker/appsettings.json +++ b/src/LiquidCode.Tester.Worker/appsettings.json @@ -86,5 +86,19 @@ "Executable": "python3.11" } } + }, + "Sandbox": { + "Enabled": true, + "DockerExecutable": "docker", + "DefaultCpuLimit": 1.0, + "DefaultMemoryLimitMb": 512, + "DefaultPidsLimit": 128, + "Languages": { + "cpp": { "ExecutionImage": "gcc:13" }, + "csharp": { "ExecutionImage": "mcr.microsoft.com/dotnet/runtime:9.0" }, + "java": { "ExecutionImage": "eclipse-temurin:17-jre" }, + "kotlin": { "ExecutionImage": "eclipse-temurin:17-jre" }, + "python": { "ExecutionImage": "python:3.11-slim" } + } } } diff --git a/tests/LiquidCode.Tester.Worker.Tests/ExecutionServiceFactoryTests.cs b/tests/LiquidCode.Tester.Worker.Tests/ExecutionServiceFactoryTests.cs index 7c59bef..5029cbc 100644 --- a/tests/LiquidCode.Tester.Worker.Tests/ExecutionServiceFactoryTests.cs +++ b/tests/LiquidCode.Tester.Worker.Tests/ExecutionServiceFactoryTests.cs @@ -1,4 +1,5 @@ using LiquidCode.Tester.Worker.Services; +using LiquidCode.Tester.Worker.Services.Sandbox; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -32,6 +33,7 @@ public class ExecutionServiceFactoryTests services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(_ => Mock.Of()); _serviceProvider = services.BuildServiceProvider(); _factory = new ExecutionServiceFactory( diff --git a/tests/LiquidCode.Tester.Worker.Tests/LiquidCode.Tester.Worker.Tests.csproj b/tests/LiquidCode.Tester.Worker.Tests/LiquidCode.Tester.Worker.Tests.csproj index 0539338..c7e8756 100644 --- a/tests/LiquidCode.Tester.Worker.Tests/LiquidCode.Tester.Worker.Tests.csproj +++ b/tests/LiquidCode.Tester.Worker.Tests/LiquidCode.Tester.Worker.Tests.csproj @@ -14,6 +14,7 @@ + diff --git a/tests/LiquidCode.Tester.Worker.Tests/PolygonPackageIntegrationTests.cs b/tests/LiquidCode.Tester.Worker.Tests/PolygonPackageIntegrationTests.cs index 139c5d5..61c1d16 100644 --- a/tests/LiquidCode.Tester.Worker.Tests/PolygonPackageIntegrationTests.cs +++ b/tests/LiquidCode.Tester.Worker.Tests/PolygonPackageIntegrationTests.cs @@ -1,10 +1,13 @@ +using System; using System.Diagnostics; using System.IO; using System.IO.Compression; using LiquidCode.Tester.Worker.Services; +using LiquidCode.Tester.Worker.Services.Sandbox; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Xunit; namespace LiquidCode.Tester.Worker.Tests; @@ -12,28 +15,45 @@ namespace LiquidCode.Tester.Worker.Tests; public class PolygonPackageIntegrationTests { private readonly PackageParserService _parserService; + private readonly SandboxOptions _sandboxOptions; public PolygonPackageIntegrationTests() { - var configuration = new ConfigurationBuilder().Build(); + var configuration = BuildConfiguration(); - var cppCompilation = new CppCompilationService(NullLogger.Instance, configuration); - var cppExecution = new CppExecutionService(NullLogger.Instance); + var loggerFactory = LoggerFactory.Create(builder => + { + builder.SetMinimumLevel(LogLevel.Debug); + builder.AddConsole(); + }); + + _sandboxOptions = new SandboxOptions(); + configuration.GetSection("Sandbox").Bind(_sandboxOptions); + + if (!_sandboxOptions.Enabled) + { + _sandboxOptions.Enabled = true; + } + + var cppCompilation = new CppCompilationService(loggerFactory.CreateLogger(), configuration); + var sandbox = new ExecutionSandbox(Options.Create(_sandboxOptions), loggerFactory.CreateLogger()); + var cppExecution = new CppExecutionService(sandbox, loggerFactory.CreateLogger()); var services = new ServiceCollection(); services.AddSingleton(configuration); services.AddSingleton(cppCompilation); + services.AddSingleton(sandbox); services.AddSingleton(cppExecution); var serviceProvider = services.BuildServiceProvider(); - var compilationFactory = new CompilationServiceFactory(serviceProvider, NullLogger.Instance); - var executionFactory = new ExecutionServiceFactory(serviceProvider, NullLogger.Instance); - var answerGenerator = new AnswerGenerationService(compilationFactory, executionFactory, NullLogger.Instance); - var polygonParser = new PolygonProblemXmlParser(NullLogger.Instance); + var compilationFactory = new CompilationServiceFactory(serviceProvider, loggerFactory.CreateLogger()); + var executionFactory = new ExecutionServiceFactory(serviceProvider, loggerFactory.CreateLogger()); + var answerGenerator = new AnswerGenerationService(compilationFactory, executionFactory, loggerFactory.CreateLogger()); + var polygonParser = new PolygonProblemXmlParser(loggerFactory.CreateLogger()); _parserService = new PackageParserService( - NullLogger.Instance, + loggerFactory.CreateLogger(), polygonParser, answerGenerator, cppCompilation); @@ -47,6 +67,20 @@ public class PolygonPackageIntegrationTests return; } + var dockerExecutable = string.IsNullOrWhiteSpace(_sandboxOptions.DockerExecutable) + ? "docker" + : _sandboxOptions.DockerExecutable; + + if (!IsExecutableAvailable(dockerExecutable)) + { + return; + } + + Assert.True( + _sandboxOptions.Languages.TryGetValue("cpp", out var cppLanguage) && + !string.IsNullOrWhiteSpace(cppLanguage.ExecutionImage), + "Sandbox configuration must specify an execution image for C++."); + var packageDirectory = ResolvePackageDirectory("exam-queue-17"); Assert.True(Directory.Exists(packageDirectory)); @@ -70,13 +104,16 @@ public class PolygonPackageIntegrationTests } finally { - if (!string.IsNullOrEmpty(package.ExtractionRoot) && Directory.Exists(package.ExtractionRoot)) + if (!KeepTemporaryExtraction()) { - Directory.Delete(package.ExtractionRoot, recursive: true); - } - else if (Directory.Exists(package.WorkingDirectory)) - { - Directory.Delete(package.WorkingDirectory, recursive: true); + if (!string.IsNullOrEmpty(package.ExtractionRoot) && Directory.Exists(package.ExtractionRoot)) + { + Directory.Delete(package.ExtractionRoot, recursive: true); + } + else if (Directory.Exists(package.WorkingDirectory)) + { + Directory.Delete(package.WorkingDirectory, recursive: true); + } } } } @@ -143,10 +180,29 @@ public class PolygonPackageIntegrationTests return memoryStream; } - private static string ResolvePackageDirectory(string folderName) + private static IConfiguration BuildConfiguration() + { + var repositoryRoot = ResolveRepositoryRoot(); + + return new ConfigurationBuilder() + .SetBasePath(repositoryRoot) + .AddJsonFile("src/LiquidCode.Tester.Worker/appsettings.json", optional: false) + .AddJsonFile("src/LiquidCode.Tester.Worker/appsettings.Development.json", optional: true) + .AddEnvironmentVariables() + .Build(); + } + + private static string ResolveRepositoryRoot() { var baseDirectory = AppContext.BaseDirectory; - var root = Path.GetFullPath(Path.Combine(baseDirectory, "..", "..", "..", "..", "..")); - return Path.Combine(root, folderName); + return Path.GetFullPath(Path.Combine(baseDirectory, "..", "..", "..", "..", "..")); } + + private static string ResolvePackageDirectory(string folderName) + { + return Path.Combine(ResolveRepositoryRoot(), folderName); + } + + private static bool KeepTemporaryExtraction() => + string.Equals(Environment.GetEnvironmentVariable("LC_KEEP_POLYGON_TEMP"), "1", StringComparison.Ordinal); }