Http oracle timeout

core-tests/e2e-tests/spring/spring-rest-bb/src/main/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutApplication.kt

@@ -0,0 +1,47 @@
+package com.foo.rest.examples.bb.httptimeout
+
+import org.evomaster.e2etests.utils.CoveredTargets
+import org.springframework.boot.SpringApplication
+import org.springframework.boot.autoconfigure.SpringBootApplication
+import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
+import org.springframework.http.ResponseEntity
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.PathVariable
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RestController
+
+
+@SpringBootApplication(exclude = [SecurityAutoConfiguration::class])
+@RequestMapping(path = ["/api/timeout"])
+@RestController
+open class BBHttpTimeoutApplication {
+
+    companion object {
+        @JvmStatic
+        fun main(args: Array<String>) {
+            SpringApplication.run(BBHttpTimeoutApplication::class.java, *args)
+        }
+    }
+
+    // slow endpoint: blocks longer than the client timeout, triggering a HTTP_TIMEOUT fault.
+    // the target is covered as soon as the request is handled, before the client gives up.
+    @GetMapping(path = ["/slow/{id}"])
+    open fun slow(@PathVariable("id") id: Int): ResponseEntity<String> {
+        CoveredTargets.cover("timeout")
+        val deadline = System.currentTimeMillis() + 10_000
+        while (System.currentTimeMillis() < deadline) {
+            try {
+                Thread.sleep(deadline - System.currentTimeMillis())
+            } catch (e: InterruptedException) {
+                // ignore and keep blocking
+            }
+        }
+        return ResponseEntity.status(200).body("$id")
+    }
+
+    // clean
+    @GetMapping(path = ["/fast/{id}"])
+    open fun fast(@PathVariable("id") id: Int): ResponseEntity<String> {
+        return ResponseEntity.status(200).body("$id")
+    }
+}

core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/com/foo/rest/examples/bb/httptimeout/BBHttpTimeoutController.kt

@@ -0,0 +1,5 @@
+package com.foo.rest.examples.bb.httptimeout
+
+import com.foo.rest.examples.bb.SpringController
+
+class BBHttpTimeoutController : SpringController(BBHttpTimeoutApplication::class.java)

core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/httptimeout/BBHttpTimeoutEMTest.kt

@@ -0,0 +1,58 @@
+package org.evomaster.e2etests.spring.rest.bb.httptimeout
+
+import com.foo.rest.examples.bb.httptimeout.BBHttpTimeoutController
+import org.evomaster.core.output.OutputFormat
+import org.evomaster.core.problem.enterprise.DetectedFaultUtils
+import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory
+import org.evomaster.e2etests.spring.rest.bb.SpringTestBase
+import org.evomaster.e2etests.utils.EnterpriseTestBase
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.EnumSource
+
+class BBHttpTimeoutEMTest : SpringTestBase() {
+
+    companion object {
+
+        init {
+            EnterpriseTestBase.shouldApplyInstrumentation = false
+        }
+
+        @BeforeAll
+        @JvmStatic
+        fun init() {
+            initClass(BBHttpTimeoutController())
+        }
+    }
+
+    @ParameterizedTest
+    @EnumSource
+    fun testBlackBoxOutput(outputFormat: OutputFormat) {
+
+        executeAndEvaluateBBTest(
+            outputFormat,
+            "bbhttptimeout",
+            5,
+            6,
+            "timeout"
+        ) { args: MutableList<String> ->
+
+            setOption(args, "useExperimentalOracles", "true")
+            setOption(args, "tcpTimeoutMs", "2000")
+            setOption(args, "httpOracles", "true")
+
+            val solution = initAndRun(args)
+
+            assertTrue(solution.individuals.size >= 1)
+
+            val timeoutFaults = DetectedFaultUtils.getDetectedFaults(solution)
+                .filter { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT }
+
+            // fault on the slow path
+            assertTrue(timeoutFaults.any { it.operationId.contains("/api/timeout/slow/") })
+            // no false positive on the fast path
+            assertTrue(timeoutFaults.none { it.operationId.contains("/api/timeout/fast/") })
+        }
+    }
+}

core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt

@@ -13,6 +13,8 @@ import org.evomaster.core.output.dto.DtoCall
 import org.evomaster.core.output.dto.GeneToDto
 import org.evomaster.core.output.formatter.OutputFormatter
 import org.evomaster.core.problem.enterprise.EnterpriseActionGroup
+import org.evomaster.core.problem.enterprise.EnterpriseActionResult
+import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory
 import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceAction
 import org.evomaster.core.problem.httpws.HttpWsAction
 import org.evomaster.core.problem.httpws.HttpWsCallResult
@@ -60,6 +62,21 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() {
         return !(result as HttpWsCallResult).getTimedout()
     }
 
+    /**
+     * For a HTTP_TIMEOUT fault, the call is expected to fail, and the assertion is expressed
+     * directly on the call:
+     *  - JS:     await expect(...).rejects.toThrow()
+     *  - Python: with self.assertRaises(Exception): ...
+     */
+    protected fun hasTimeoutFault(res: ActionResult): Boolean {
+        return res is EnterpriseActionResult
+                && res.getFaults().any { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT }
+    }
+
+    protected fun expectsRejection(res: ActionResult) = format.isJavaScript() && hasTimeoutFault(res)
+
+    protected fun expectsAssertRaises(res: ActionResult) = format.isPython() && hasTimeoutFault(res)
+
     fun startRequest(lines: Lines){
         when {
             format.isJavaOrKotlin() -> lines.append("given()")
@@ -114,7 +131,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() {
 
         when {
             format.isJavaOrKotlin() -> lines.append("given()")
-            format.isJavaScript() -> lines.append("await superagent")
+            // for a call expected to reject, wrap it in await expect(...).rejects.toThrow()
+            format.isJavaScript() -> lines.append(if (expectsRejection(res)) "await expect(superagent" else "await superagent")
             format.isCsharp() -> lines.append("await Client")
             format.isPython() -> lines.append("requests \\")
         }
@@ -398,7 +416,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() {
             timeStartName = handleExecutionTimePrologue(lines);
         }
 
-        if (res.invalidCall()) {
+        if (res.invalidCall() && !expectsRejection(res) && !expectsAssertRaises(res)) {
             addActionInTryCatch(call, index, testCaseName, lines, res, testSuitePath, baseUrlOfSut)
         } else {
             addActionLines(call, index, testCaseName, lines, res, testSuitePath, baseUrlOfSut)
@@ -502,6 +520,12 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() {
             dtoVar = writeDto(call, lines)
         }
 
+        val pyAssertRaises = expectsAssertRaises(res)
+        if (pyAssertRaises) {
+            lines.add("with self.assertRaises(Exception):")
+            lines.indent()
+        }
+
         handleFirstLine(call, lines, res, responseVariableName)
 
         when {
@@ -516,6 +540,8 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() {
                 lines.indent(2)
                 //in SuperAgent, verb must be first
                 handleVerbEndpoint(baseUrlOfSut, call, lines)
+                //client timeout, same source as fuzzing tcpTimeoutMs
+                lines.add(".timeout({response: ${TestSuiteWriter.httpTimeoutVarMs}, deadline: ${TestSuiteWriter.httpTimeoutVarMs}})")
                 lines.append(getAcceptHeader(call, res))
                 handleHeaders(call, lines)
                 handleBody(call, lines)
@@ -537,6 +563,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() {
             handleResponseDirectlyInTheCall(call, res, lines)
         }
         handleLastLine(call, res, lines, responseVariableName)
+
+        if (pyAssertRaises) {
+            lines.deindent()
+        }
         return responseVariableName
     }
 
@@ -980,6 +1010,10 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() {
                 so, here we make it passes as long as a status was present
              */
             lines.add(".ok(res => res.status)")
+            if (expectsRejection(res)) {
+                //close the await expect(...) and assert the call rejected
+                lines.append(").rejects.toThrow()")
+            }
         }
 
 

core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt

@@ -303,11 +303,12 @@ class RestTestCaseWriter : HttpWsTestCaseWriter {
                 if (bodyParam != null) {
                     lines.append(", data=body")
                 }
-                if(config.testTimeout > 0) {
+                if(config.tcpTimeoutMs > 0) {
                     /*
-                        As timeout at test level does not work reliably in Python, we do timeout as well in each HTTP call.
+                        Client timeout per HTTP call, same source as fuzzing tcpTimeoutMs.
+                        Also, timeout at test level does not work reliably in Python.
                     */
-                    lines.append(", timeout=${config.testTimeout}")
+                    lines.append(", timeout=${TestSuiteWriter.httpTimeoutVarSeconds}")
                 }
             }
         }

core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt

@@ -9,6 +9,7 @@ import org.evomaster.core.output.TestWriterUtils
 import org.evomaster.core.output.TestWriterUtils.getWireMockVariableName
 import org.evomaster.core.problem.enterprise.EnterpriseActionResult
 import org.evomaster.core.problem.enterprise.EnterpriseIndividual
+import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory
 import org.evomaster.core.problem.externalservice.HostnameResolutionAction
 import org.evomaster.core.problem.externalservice.httpws.HttpExternalServiceAction
 import org.evomaster.core.problem.externalservice.httpws.param.HttpWsResponseParam
@@ -44,6 +45,13 @@ abstract class TestCaseWriter {
 
     companion object {
         private val log = LoggerFactory.getLogger(TestCaseWriter::class.java)
+
+
+        /**
+        * message for the assertion that flags a missing expected timeout (Java/Kotlin/C#)
+        * JS uses await expect(...).rejects.toThrow() and Python uses with self.assertRaises(...)
+        */
+        private const val EXPECTED_TIMEOUT_MSG = "Expected a timeout"
     }
 
 
@@ -320,10 +328,19 @@ abstract class TestCaseWriter {
             format.isPython() -> lines.add("try:")
         }
 
+        // a HTTP_TIMEOUT fault means the call is expected to time out (client timeout == fuzzing
+        // tcpTimeoutMs). if no timeout exception is thrown, the fault did not reproduce -> fail
+        val timeoutFault = res is EnterpriseActionResult
+                && res.getFaults().any { it.category == ExperimentalFaultCategory.HTTP_TIMEOUT }
+
         lines.indented {
             addActionLines(call,index, testCaseName, lines, res, testSuitePath, baseUrlOfSut)
 
-            if (shouldFailIfExceptionNotThrown(res)) {
+            if (timeoutFault) {
+                // only Java/Kotlin/C# reach here; JS uses expect(...).rejects.toThrow() and
+                // Python uses with self.assertRaises(...), neither wrapped in this try/catch
+                lines.add("fail(\"$EXPECTED_TIMEOUT_MSG\");")
+            } else if (shouldFailIfExceptionNotThrown(res)) {
                 if (!format.isJavaScript()) {
                     /*
                         TODO need a way to do it for JS, see
@@ -372,17 +389,16 @@ abstract class TestCaseWriter {
             format.isPython() -> lines.add("except Exception as e:")
         }
 
-        res.getErrorMessage()?.let {
-            lines.indented {
+        lines.indented {
+            res.getErrorMessage()?.let {
                 lines.addSingleCommentLine("${it.replace('\n', ' ').replace('\r', ' ')}")
             }
-        }
-
-        if (format.isPython()) {
-            lines.indented {
+            if (format.isPython()) {
                 lines.add("pass")
             }
-        } else {
+        }
+
+        if (!format.isPython()) {
             lines.add("}")
         }
     }

core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt

@@ -58,6 +58,14 @@ class TestSuiteWriter {
         const val pythonUtilsFilename = "$pythonUtilsFilenameNoExtension.py"
         const val javascriptUtilsFilename = "EMTestUtils.js"
 
+        /**
+         * Names of the variables, generated into each test suite, holding the per-HTTP-call
+         * client timeout. Same source as the fuzzing 'tcpTimeoutMs' option. Two unit-specific
+         * variants, as different output formats need different units.
+         */
+        const val httpTimeoutVarMs = "EM_HTTP_TIMEOUT_MS"     // milliseconds (JS/superagent)
+        const val httpTimeoutVarSeconds = "EM_HTTP_TIMEOUT_S" // seconds (Python/requests)
+
         private val log: Logger = LoggerFactory.getLogger(TestSuiteWriter::class.java)
 
         const val baseUrlOfSut = "baseUrlOfSut"
@@ -465,6 +473,7 @@ class TestSuiteWriter {
                 addImport("io.restassured.RestAssured", lines)
                 addImport("io.restassured.RestAssured.given", lines, true)
                 addImport("io.restassured.response.ValidatableResponse", lines)
+                addImport("io.restassured.config.HttpClientConfig", lines)
             }
 
             if ((config.isEnabledExternalServiceMocking() && solution.needWireMockServers())
@@ -533,6 +542,8 @@ class TestSuiteWriter {
         if (format.isJavaScript()) {
             lines.add("process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';")
             lines.add("const superagent = require(\"superagent\");")
+            // HTTP client timeout (ms)
+            lines.add("const $httpTimeoutVarMs = ${config.tcpTimeoutMs};")
 
             val jsUtils = JsLoader::class.java.getResource("/$javascriptUtilsFilename").readText()
             saveToDisk(jsUtils, Paths.get(config.outputFolder, javascriptUtilsFilename))
@@ -587,6 +598,8 @@ class TestSuiteWriter {
                 }
             }
             lines.add("from $pythonUtilsFilenameNoExtension import *")
+            // HTTP client timeout (seconds)
+            lines.add("$httpTimeoutVarSeconds = ${config.tcpTimeoutMs / 1000.0}")
             val pythonUtils = PyLoader::class.java.getResource("/$pythonUtilsFilename").readText()
             saveToDisk(pythonUtils, Paths.get(config.outputFolder, pythonUtilsFilename))
         }
@@ -840,9 +853,20 @@ class TestSuiteWriter {
                     addStatement("RestAssured.urlEncodingEnabled = false", lines)
                 }
 
-                if (config.enableBasicAssertions && format.isJavaOrKotlin()) {
+                if (format.isJavaOrKotlin()) {
+                    // global HTTP client config. The socket timeout MUST match the one used during
+                    // fuzzing (tcpTimeoutMs), so that timeout faults are reproduced consistently
                     lines.add("RestAssured.config = RestAssured.config()")
                     lines.indented {
+                        if (config.enableBasicAssertions) {
+                            lines.add(".jsonConfig(JsonConfig.jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))")
+                            lines.add(".redirect(redirectConfig().followRedirects(false))")
+                        }
+                        lines.add(".httpClient(HttpClientConfig.httpClientConfig()")
+                        lines.indented {
+                            lines.add(".setParam(\"http.socket.timeout\", ${config.tcpTimeoutMs})")
+                            lines.add(".setParam(\"http.connection.timeout\", ${config.tcpTimeoutMs}))")
+                        }
                         lines.add(".jsonConfig(JsonConfig.jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))")
                         lines.add(".redirect(redirectConfig().followRedirects(false))")
                         lines.add(".encoderConfig(EncoderConfig.encoderConfig().encodeContentTypeAs(\"application/octet-stream\", ContentType.TEXT))")

core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt

@@ -31,6 +31,8 @@ enum class ExperimentalFaultCategory(
         "TODO"),
     HTTP_INVALID_ALLOW(919, "Invalid allow", "invalidAllow",
         "TODO"),
+    HTTP_TIMEOUT(921, "Timeout", "timeout",
+        "TODO"),
 
     HTTP_STATUS_NO_NON_STANDARD_CODES(950, "HTTP/REST-Design Violation: no-non-standard-codes", "invalidStatusCode", "TODO"),
     HTTP_STATUS_NO_201_IF_DELETE(951, "HTTP/REST-Design Violation: no-201-if-delete", "201OnDelete",  "TODO"),

core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt

@@ -1312,6 +1312,36 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
             analyzeHttpSemantics(individual, actionResults, fv)
         }
 
+
+//         This oracle should only be considered for Black-Box testing.
+//         In White-Box testing, instrumentation may affect execution time,
+//         especially for CPU-bound APIs.
+        if(config.blackBox && config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_TIMEOUT)){
+            handleTimeout(individual, actionResults, fv)
+        }
+    }
+
+    /**
+     * A timeout is treated as a fault: the SUT should rather answer quickly (eg 202 with a
+     * Location header for long computations) instead of hanging until the client gives up.
+     */
+    private fun handleTimeout(
+        individual: RestIndividual,
+        actionResults: List<ActionResult>,
+        fv: FitnessValue
+    ) {
+        val actions = individual.seeMainExecutableActions()
+
+        for (index in actions.indices) {
+            val a = actions[index]
+            val r = actionResults.find { it.sourceLocalId == a.getLocalId() } as RestCallResult? ?: continue
+            if (!r.getTimedout()) continue
+
+            val category = ExperimentalFaultCategory.HTTP_TIMEOUT
+            val scenarioId = idMapper.handleLocalTarget(idMapper.getFaultDescriptiveId(category, a.getName()))
+            fv.updateTarget(scenarioId, 1.0, index)
+            r.addFault(DetectedFault(category, a.getName(), null))
+        }
     }
 
     private fun analyzeHttpSemantics(individual: RestIndividual, actionResults: List<ActionResult>, fv: FitnessValue) {