fix memory limiting java & kotlin

src/LiquidCode.Tester.Worker/Services/JavaExecutionServiceIsolate.cs

@@ -69,16 +69,25 @@ public class JavaExecutionServiceIsolate : IExecutionService
 
             // Run in Isolate
             // Note: Java needs more memory for JVM overhead
-            var javaMemoryMb = Math.Max(memoryLimitMb, 128); // Minimum 128MB for JVM
+            // We set JVM heap limit to the requested memory, but give Isolate more for JVM internals
+            var jvmHeapMb = memoryLimitMb;
+            var jvmTotalMemoryMb = memoryLimitMb + 64; // Add 64MB for JVM overhead (metaspace, code cache, etc.)
+
+            var arguments = new List<string>
+            {
+                $"-Xmx{jvmHeapMb}m",  // Max heap size = requested memory limit
+                $"-Xms{Math.Min(jvmHeapMb, 32)}m",  // Initial heap size (min 32MB or requested)
+                "-cp", "/box", "Solution"
+            };
 
             var isolateResult = await _isolateService.RunAsync(new IsolateRunOptions
             {
                 BoxId = boxId,
                 Executable = "/usr/bin/java",
-                Arguments = new[] { "-cp", "/box", "Solution" },
+                Arguments = arguments.ToArray(),
                 TimeLimitSeconds = timeLimitMs / 1000.0,
                 WallTimeLimitSeconds = (timeLimitMs / 1000.0) * 2,
-                MemoryLimitKb = javaMemoryMb * 1024,
+                MemoryLimitKb = jvmTotalMemoryMb * 1024,
                 StackLimitKb = 256 * 1024, // 256 MB stack
                 ProcessLimit = 64, // Java creates multiple threads
                 EnableNetwork = false,

src/LiquidCode.Tester.Worker/Services/KotlinExecutionServiceIsolate.cs

@@ -64,16 +64,25 @@ public class KotlinExecutionServiceIsolate : IExecutionService
             }
 
             // Run in Isolate (Kotlin runs via Java)
-            var kotlinMemoryMb = Math.Max(memoryLimitMb, 128); // Minimum 128MB for JVM
+            // We set JVM heap limit to the requested memory, but give Isolate more for JVM internals
+            var jvmHeapMb = memoryLimitMb;
+            var jvmTotalMemoryMb = memoryLimitMb + 64; // Add 64MB for JVM overhead
+
+            var arguments = new List<string>
+            {
+                $"-Xmx{jvmHeapMb}m",  // Max heap size = requested memory limit
+                $"-Xms{Math.Min(jvmHeapMb, 32)}m",  // Initial heap size
+                "-jar", $"/box/{jarName}"
+            };
 
             var isolateResult = await _isolateService.RunAsync(new IsolateRunOptions
             {
                 BoxId = boxId,
                 Executable = "/usr/bin/java",
-                Arguments = new[] { "-jar", $"/box/{jarName}" },
+                Arguments = arguments.ToArray(),
                 TimeLimitSeconds = timeLimitMs / 1000.0,
                 WallTimeLimitSeconds = (timeLimitMs / 1000.0) * 2,
-                MemoryLimitKb = kotlinMemoryMb * 1024,
+                MemoryLimitKb = jvmTotalMemoryMb * 1024,
                 StackLimitKb = 256 * 1024,
                 ProcessLimit = 64, // JVM creates multiple threads
                 EnableNetwork = false,