From 7d92741d7f77df882a9d7706bf61b9dcd0c3cca4 Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Tue, 12 Dec 2023 14:46:43 +0300 Subject: [PATCH 01/10] Work for sbft --- .../workflows/publish-plugin-from-branch.yml | 1 - build.gradle.kts | 41 +- gradlew | 0 .../utbot/cli/language/python/Application.kt | 10 +- .../python/PythonGenerateTestsCommand.kt | 19 +- .../language/python/sbft/SbftCliProcessor.kt | 15 + .../python/sbft/SbftGenerateTestsCommand.kt | 291 +++++++++++ .../utbot/cli/language/python/sbft/Utils.kt | 20 + utbot-intellij-python/build.gradle.kts | 4 +- .../plugin/python/PythonDialogProcessor.kt | 24 +- .../org/utbot/intellij/plugin/python/Utils.kt | 10 +- .../plugin/settings/SettingsWindow.kt | 2 +- .../src/main/python/utbot_executor/README.md | 2 + .../main/python/utbot_executor/pyproject.toml | 2 +- .../python/utbot_executor/tests/pytest.ini | 2 - .../tests/test_deep_serialization.py | 118 +++-- .../utbot_executor/tests/test_executor.py | 53 ++ .../tests/test_python_executor.py | 33 ++ .../utbot_executor/utbot_executor/__main__.py | 27 +- .../utbot_executor/utbot_executor/config.py | 30 ++ .../deep_serialization/iterator_wrapper.py | 74 +++ .../deep_serialization/json_converter.py | 53 +- .../deep_serialization/memory_objects.py | 80 ++- .../deep_serialization/utils.py | 4 +- .../utbot_executor/example/example.py | 2 +- .../utbot_executor/utbot_executor/executor.py | 259 +++++++--- .../utbot_executor/utbot_executor/listener.py | 67 +-- .../utbot_executor/memory_compressor.py | 7 +- .../utbot_executor/utbot_executor/parser.py | 18 +- .../utbot_executor/ut_tracer.py | 62 ++- .../utbot_executor/utbot_executor/utils.py | 21 +- .../src/main/resources/utbot_executor_version | 2 +- .../org/utbot/python/newtyping/PythonType.kt | 32 +- .../utbot/python/newtyping/mypy/RunMypy.kt | 2 +- .../python/utbot_mypy_runner/pyproject.toml | 2 +- .../utbot_mypy_runner/extract_annotations.py | 18 +- .../utbot_mypy_runner/nodes.py | 30 +- .../main/resources/utbot_mypy_runner_version | 2 +- utbot-python/build.gradle.kts | 7 +- .../samples/algorithms/floyd_warshall.py | 42 ++ .../samples/samples/easy_samples/my_func.py | 6 + .../kotlin/org/utbot/python/PythonEngine.kt | 386 --------------- .../utbot/python/PythonTestCaseGenerator.kt | 342 ++----------- .../python/PythonTestGenerationConfig.kt | 37 +- .../python/PythonTestGenerationProcessor.kt | 156 ++++-- .../kotlin/org/utbot/python/UTPythonAPI.kt | 259 +++++++++- .../org/utbot/python/code/PythonCodeAPI.kt | 11 +- .../org/utbot/python/coverage/CoverageApi.kt | 23 +- .../utbot/python/engine/ExecutionFeedback.kt | 12 + .../utbot/python/engine/ExecutionStorage.kt | 36 ++ .../utbot/python/engine/GlobalPythonEngine.kt | 167 +++++++ .../python/engine/fuzzing/FuzzingEngine.kt | 465 ++++++++++++++++++ .../typeinference/FunctionAnnotationUtils.kt | 91 ++++ .../python/engine/symbolic/SymbolicEngine.kt | 57 +++ .../symbolic/SymbolicExecutionEvaluator.kt | 195 ++++++++ .../USVMPythonAnalysisResultReceiverImpl.kt | 16 + .../engine/utils/ModelsTransformation.kt | 31 ++ .../python/evaluation/CodeEvaluationApi.kt | 5 + .../evaluation/PythonCodeSocketExecutor.kt | 67 ++- .../python/evaluation/PythonWorkerManager.kt | 68 ++- .../python/evaluation/UtExecutorThread.kt | 7 + .../ExecutionRequestSerializer.kt | 6 + .../ExecutionResultDeserializer.kt | 7 +- .../serialization/PythonObjectParser.kt | 29 +- .../python/framework/api/python/PythonApi.kt | 16 +- .../python/framework/api/python/PythonTree.kt | 30 +- .../api/python/util/PythonIdUtils.kt | 2 + .../framework/api/python/util/PythonUtils.kt | 4 +- .../codegen/model/PythonCodeGenerator.kt | 4 +- .../tree/PythonCgMethodConstructor.kt | 92 +++- .../tree/PythonCgVariableConstructor.kt | 22 +- .../tree/PythonTestFrameworkManager.kt | 12 +- .../constructor/visitor/CgPythonRenderer.kt | 14 +- .../constructor/visitor/CgPythonVisitor.kt | 2 + .../codegen/model/tree/CgPythonElement.kt | 33 +- .../framework/external/PythonUtBotJavaApi.kt | 4 +- .../org/utbot/python/fuzzing/PythonApi.kt | 125 +++-- .../fuzzing/provider/BoolValueProvider.kt | 13 +- .../provider/BytearrayValueProvider.kt | 15 +- .../fuzzing/provider/BytesValueProvider.kt | 15 +- .../fuzzing/provider/ComplexValueProvider.kt | 57 ++- .../fuzzing/provider/ConstantValueProvider.kt | 11 +- .../fuzzing/provider/DictValueProvider.kt | 55 +-- .../fuzzing/provider/FloatValueProvider.kt | 13 +- .../fuzzing/provider/IntValueProvider.kt | 14 +- .../fuzzing/provider/IteratorValueProvider.kt | 37 ++ .../fuzzing/provider/ListValueProvider.kt | 18 +- .../fuzzing/provider/NoneValueProvider.kt | 12 +- .../fuzzing/provider/OptionalValueProvider.kt | 30 ++ .../provider/RePatternValueProvider.kt | 20 +- .../fuzzing/provider/ReduceValueProvider.kt | 233 ++++++--- .../fuzzing/provider/SetValueProvider.kt | 47 +- .../fuzzing/provider/StrValueProvider.kt | 19 +- .../fuzzing/provider/SubtypeValueProvider.kt | 30 +- .../provider/TupleFixSizeValueProvider.kt | 23 +- .../fuzzing/provider/TupleValueProvider.kt | 36 +- .../provider/TypeAliasValueProvider.kt | 22 +- .../fuzzing/provider/UnionValueProvider.kt | 19 +- .../fuzzing/provider/utils/ProviderUtils.kt | 21 +- .../utbot/python/newtyping/ast/ParseUtils.kt | 27 +- .../ast/visitor/hints/HintCollector.kt | 9 +- .../newtyping/ast/visitor/hints/Protocols.kt | 6 - .../inference/TypeInferenceProcessor.kt | 34 +- .../inference/baseline/BaselineAlgorithm.kt | 98 ++-- .../inference/baseline/StateExpansion.kt | 5 +- .../inference/baseline/Structures.kt | 3 +- .../utbot/python/newtyping/mypy/RunDMypy.kt | 2 +- .../utils/TestGenerationLimitManager.kt | 12 +- .../org/utbot/python/utils/TimeoutUtils.kt | 31 ++ 109 files changed, 3792 insertions(+), 1522 deletions(-) mode change 100644 => 100755 gradlew create mode 100644 utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftCliProcessor.kt create mode 100644 utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt create mode 100644 utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/Utils.kt delete mode 100644 utbot-python-executor/src/main/python/utbot_executor/tests/pytest.ini create mode 100644 utbot-python-executor/src/main/python/utbot_executor/tests/test_executor.py create mode 100644 utbot-python-executor/src/main/python/utbot_executor/tests/test_python_executor.py create mode 100644 utbot-python-executor/src/main/python/utbot_executor/utbot_executor/config.py create mode 100644 utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/iterator_wrapper.py create mode 100644 utbot-python/samples/samples/algorithms/floyd_warshall.py create mode 100644 utbot-python/samples/samples/easy_samples/my_func.py delete mode 100644 utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/engine/ExecutionFeedback.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/engine/ExecutionStorage.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/engine/fuzzing/FuzzingEngine.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/engine/fuzzing/typeinference/FunctionAnnotationUtils.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/SymbolicEngine.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/SymbolicExecutionEvaluator.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/USVMPythonAnalysisResultReceiverImpl.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/engine/utils/ModelsTransformation.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IteratorValueProvider.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/OptionalValueProvider.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/utils/TimeoutUtils.kt diff --git a/.github/workflows/publish-plugin-from-branch.yml b/.github/workflows/publish-plugin-from-branch.yml index 51415412d6..96d9761aa0 100644 --- a/.github/workflows/publish-plugin-from-branch.yml +++ b/.github/workflows/publish-plugin-from-branch.yml @@ -87,7 +87,6 @@ jobs: cd ${{ matrix.configuration.directory }}/build/distributions unzip ${{ matrix.configuration.directory }}-${{ env.VERSION }}.zip rm ${{ matrix.configuration.directory }}-${{ env.VERSION }}.zip - - name: Archive UTBot IntelliJ IDEA plugin if: ${{ inputs.upload-artifact == 'true' }} uses: actions/upload-artifact@v3 diff --git a/build.gradle.kts b/build.gradle.kts index 823b6c22ba..7322c60014 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,4 @@ import java.text.SimpleDateFormat -import org.gradle.api.JavaVersion.* import org.jetbrains.kotlin.gradle.tasks.KotlinCompile group = "org.utbot" @@ -137,6 +136,13 @@ allprojects { } } +// from GRADLE_USER_HOME/gradle.properties + val githubUserFromHome: String? by project + val githubTokenFromHome: String? by project // with permission to read packages + + val githubUser: String = githubUserFromHome ?: System.getenv("GITHUB_ACTOR") ?: error("githubUser not defined") + val githubToken: String = githubTokenFromHome ?: System.getenv("GITHUB_TOKEN") ?: error("githubToken not defined") + repositories { mavenCentral() maven("https://jitpack.io") @@ -144,6 +150,13 @@ allprojects { maven("https://plugins.gradle.org/m2") maven("https://www.jetbrains.com/intellij-repository/releases") maven("https://cache-redirector.jetbrains.com/maven-central") + maven { + url = uri("https://maven.pkg.github.com/UnitTestBot/usvm") + credentials { + username = githubUser + password = githubToken + } + } } dependencies { @@ -209,3 +222,29 @@ configure( } } } + +// from GRADLE_USER_HOME/gradle.properties +val githubUserFromHome: String? by project +val githubTokenFromHome: String? by project // with permission to read packages + +val githubUser: String = githubUserFromHome ?: System.getenv("GITHUB_ACTOR") ?: error("githubUser not defined") +val githubToken: String = githubTokenFromHome ?: System.getenv("GITHUB_TOKEN") ?: error("githubToken not defined") + +configure( + listOf( + project(":utbot-python"), + project(":utbot-intellij-python"), + project(":utbot-intellij-main"), + project(":utbot-cli-python"), + ) +) { + repositories { + maven { + url = uri("https://maven.pkg.github.com/UnitTestBot/usvm") + credentials { + username = githubUser + password = githubToken + } + } + } +} \ No newline at end of file diff --git a/gradlew b/gradlew old mode 100644 new mode 100755 diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/Application.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/Application.kt index 0cad4e6264..3fc8c69113 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/Application.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/Application.kt @@ -10,6 +10,7 @@ import mu.KotlinLogging import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.core.config.Configurator import org.slf4j.event.Level +import org.utbot.cli.language.python.sbft.SbftGenerateTestsCommand import java.util.* import kotlin.system.exitProcess @@ -29,10 +30,13 @@ class UtBotPythonCli : CliktCommand(name = "UnitTestBot Python Command Line Inte fun main(args: Array) = try { UtBotPythonCli().subcommands( - PythonGenerateTestsCommand(), - PythonRunTestsCommand(), - PythonTypeInferenceCommand() + SbftGenerateTestsCommand() ).main(args) +// UtBotPythonCli().subcommands( +// PythonGenerateTestsCommand(), +// PythonRunTestsCommand(), +// PythonTypeInferenceCommand() +// ).main(args) } catch (ex: Throwable) { ex.printStackTrace() exitProcess(1) diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt index 43b081d4a7..face9bd6bc 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonGenerateTestsCommand.kt @@ -15,6 +15,7 @@ import org.utbot.python.PythonMethodHeader import org.utbot.python.PythonTestGenerationConfig import org.utbot.python.PythonTestSet import org.utbot.python.TestFileInformation +import org.utbot.python.UsvmConfig import org.utbot.python.utils.RequirementsInstaller import org.utbot.python.code.PythonCode import org.utbot.python.coverage.PythonCoverageMode @@ -130,6 +131,17 @@ class PythonGenerateTestsCommand : CliktCommand( .choice("INSTRUCTIONS", "LINES") .default("LINES") + private val javaCmd by option( + "--java-cmd", + help = "(required) Path to Java command (ONLY FOR USVM)." + ).required() + + private val usvmDirectory by option( + "--usvm-dir", + help = "(required) Path to usvm directory (ONLY FOR USVM)." + ).required() + + private val testFramework: TestFramework get() = when (testFrameworkAsString) { @@ -269,6 +281,7 @@ class PythonGenerateTestsCommand : CliktCommand( coverageMeasureMode = PythonCoverageMode.parse(coverageMeasureMode), sendCoverageContinuously = !doNotSendCoverageContinuously, coverageOutputFormat = CoverageOutputFormat.parse(coverageOutputFormat), + usvmConfig = UsvmConfig(javaCmd, usvmDirectory) ) val processor = PythonCliProcessor( @@ -280,10 +293,10 @@ class PythonGenerateTestsCommand : CliktCommand( ) logger.info("Loading information about Python types...") - val (mypyStorage, _) = processor.sourceCodeAnalyze() + val mypyConfig = processor.sourceCodeAnalyze() logger.info("Generating tests...") - var testSets = processor.testGenerate(mypyStorage) + var testSets = processor.testGenerate(mypyConfig) if (testSets.isEmpty()) return if (doNotGenerateRegressionSuite) { testSets = testSets.map { testSet -> @@ -291,8 +304,6 @@ class PythonGenerateTestsCommand : CliktCommand( testSet.method, testSet.executions.filterNot { it.result is UtExecutionSuccess }, testSet.errors, - testSet.mypyReport, - testSet.classId, testSet.executionsNumber ) } diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftCliProcessor.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftCliProcessor.kt new file mode 100644 index 0000000000..051af36645 --- /dev/null +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftCliProcessor.kt @@ -0,0 +1,15 @@ +package org.utbot.cli.language.python.sbft + +import org.utbot.python.PythonTestGenerationConfig +import org.utbot.python.PythonTestGenerationProcessor +import org.utbot.python.PythonTestSet + +class SbftCliProcessor( + override val configuration: PythonTestGenerationConfig, +) : PythonTestGenerationProcessor() { + override fun saveTests(testsCode: String) { } + + override fun notGeneratedTestsAction(testedFunctions: List) {} + + override fun processCoverageInfo(testSets: List) {} +} diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt new file mode 100644 index 0000000000..bd873cc00f --- /dev/null +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt @@ -0,0 +1,291 @@ +package org.utbot.cli.language.python.sbft + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.options.required +import com.github.ajalt.clikt.parameters.options.split +import com.github.ajalt.clikt.parameters.types.choice +import com.github.ajalt.clikt.parameters.types.long +import mu.KotlinLogging +import org.parsers.python.PythonParser +import org.utbot.cli.language.python.CliRequirementsInstaller +import org.utbot.cli.language.python.findCurrentPythonModule +import org.utbot.cli.language.python.toAbsolutePath +import org.utbot.cli.language.python.writeToFileAndSave +import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour +import org.utbot.python.MypyConfig +import org.utbot.python.PythonMethodHeader +import org.utbot.python.PythonTestGenerationConfig +import org.utbot.python.PythonTestGenerationProcessor +import org.utbot.python.TestFileInformation +import org.utbot.python.UsvmConfig +import org.utbot.python.code.PythonCode +import org.utbot.python.coverage.CoverageOutputFormat +import org.utbot.python.coverage.PythonCoverageMode +import org.utbot.python.framework.api.python.PythonClassId +import org.utbot.python.framework.api.python.pythonBuiltinsModuleName +import org.utbot.python.framework.codegen.model.Pytest +import org.utbot.python.framework.codegen.model.PythonImport +import org.utbot.python.framework.codegen.model.PythonSysPathImport +import org.utbot.python.newtyping.ast.parseClassDefinition +import org.utbot.python.newtyping.ast.parseFunctionDefinition +import org.utbot.python.newtyping.mypy.dropInitFile +import org.utbot.python.utils.Cleaner +import org.utbot.python.utils.Fail +import org.utbot.python.utils.RequirementsInstaller +import org.utbot.python.utils.Success +import org.utbot.python.utils.separateTimeout +import java.io.File +import java.util.concurrent.atomic.AtomicBoolean +import kotlin.system.exitProcess +import kotlin.system.measureTimeMillis + +private const val DEFAULT_TIMEOUT_IN_MILLIS = 60000L +private const val DEFAULT_TIMEOUT_FOR_ONE_RUN_IN_MILLIS = 2000L + +private val logger = KotlinLogging.logger {} + +class SbftGenerateTestsCommand : CliktCommand( + name = "generate_python", + help = "Generate tests for specified Python classes or top-level functions from a specified file." +) { + private val sourceFile by argument( + help = "File with Python code to generate tests for." + ) + + private lateinit var absPathToSourceFile: String + private lateinit var sourceFileContent: String + + private val directoriesForSysPath by option( + "-s", "--sys-path", + help = "(required) Directories to add to sys.path. " + + "One of directories must contain the file with the methods under test." + ).split(",").required() + + private val pythonPath by option( + "-p", "--python-path", + help = "(required) Path to Python interpreter." + ).required() + + private val output by option( + "-o", "--output", help = "(required) File for generated tests." + ).required() + + private val doNotMinimize by option( + "--do-not-minimize", + help = "Turn off minimization of the number of generated tests." + ).flag(default = false) + + private val timeout by option( + "-t", "--timeout", + help = "Specify the maximum time in milliseconds to spend on generating tests ($DEFAULT_TIMEOUT_IN_MILLIS by default)." + ).long().default(DEFAULT_TIMEOUT_IN_MILLIS) + + private val timeoutForRun by option( + "--timeout-for-run", + help = "Specify the maximum time in milliseconds to spend on one function run ($DEFAULT_TIMEOUT_FOR_ONE_RUN_IN_MILLIS by default)." + ).long().default(DEFAULT_TIMEOUT_FOR_ONE_RUN_IN_MILLIS) + + private val includeMypyAnalysisTime by option( + "--include-mypy-analysis-time", + help = "Include mypy static analysis time in the total timeout." + ).flag(default = false) + + private val runtimeExceptionTestsBehaviour by option("--runtime-exception-behaviour", help = "PASS or FAIL") + .choice("PASS", "FAIL") + .default("FAIL") + + private val coverageMeasureMode by option("--coverage-measure-mode", help = "Use LINES or INSTRUCTIONS for coverage measurement.") + .choice("INSTRUCTIONS", "LINES") + .default("INSTRUCTIONS") + + private val doNotSendCoverageContinuously by option("--do-not-send-coverage-continuously", help = "Do not send coverage during execution.") + .flag(default = false) + + private val prohibitedExceptions by option("--prohibited-exceptions", help = "Do not generate tests with these exceptions. Set '-' to generate tests for all exceptions.") + .split(",") + .default(PythonTestGenerationConfig.defaultProhibitedExceptions) + + private val doNotGenerateStateAssertions by option( + "--do-not-generate-state-assertions", + help = "Do not generate state assertions for all functions excluding functions with None return value." + ) + .flag(default = false) + + private val javaCmd by option( + "--java-cmd", + help = "(required) Path to Java command (ONLY FOR USVM)." + ).required() + + private val usvmDirectory by option( + "--usvm-dir", + help = "(required) Path to usvm directory (ONLY FOR USVM)." + ).required() + + private val checkUsvm by option("--check-usvm", help = "Check usvm (ONLY FOR USVM).") + .flag(default = false) + + private fun getPythonMethods(): List> { + val parsedModule = PythonParser(sourceFileContent).Module() + + val topLevelFunctions = PythonCode.getTopLevelFunctions(parsedModule) + val topLevelClasses = PythonCode.getTopLevelClasses(parsedModule) + + val functions = topLevelFunctions + .mapNotNull { parseFunctionDefinition(it) } + .map { PythonMethodHeader(it.name.toString(), absPathToSourceFile, null) } + val methods = topLevelClasses + .mapNotNull { cls -> + val parsedClass = parseClassDefinition(cls) ?: return@mapNotNull null + val innerClasses = PythonCode.getInnerClasses(cls) + (listOf(parsedClass to null) + innerClasses.mapNotNull { innerClass -> parseClassDefinition(innerClass)?.let { it to parsedClass } }).map { (cls, parent) -> + PythonCode.getClassMethods(cls.body) + .mapNotNull { parseFunctionDefinition(it) } + .map { function -> + val clsName = (parent?.let { "${it.name}." } ?: "") + cls.name.toString() + val parsedClassName = PythonClassId(pythonBuiltinsModuleName, clsName) + PythonMethodHeader(function.name.toString(), absPathToSourceFile, parsedClassName) + } + } + } + .flatten() + return (methods + listOf(functions)).filter { it.isNotEmpty() } + } + + private val globalImportCollection = mutableSetOf() + private val globalCodeCollection = mutableListOf() + + private val shutdown: AtomicBoolean = AtomicBoolean(false) + private val alreadySaved: AtomicBoolean = AtomicBoolean(false) + + private val shutdownThread = + object : Thread() { + override fun run() { + shutdown.set(true) + try { + if (!alreadySaved.get()) { + saveTests() + } + } catch (_: InterruptedException) { + logger.warn { "Interrupted exception" } + } + } + } + + private fun addShutdownHook() { + Runtime.getRuntime().addShutdownHook(shutdownThread) + } + + private fun removeShutdownHook() { + Runtime.getRuntime().removeShutdownHook(shutdownThread) + } + + private fun saveTests() { + logger.info("Saving tests...") + val importCode = globalImportCollection + .filterNot { it is PythonSysPathImport } + .sortedBy { it.order } + .map { renderPythonImport(it) } + val testCode = (listOf(importCode.joinToString("\n")) + globalCodeCollection).joinToString("\n\n\n") + writeToFileAndSave(output, testCode) + + Cleaner.doCleaning() + alreadySaved.set(true) + } + + override fun run() { + absPathToSourceFile = sourceFile.toAbsolutePath() + sourceFileContent = File(absPathToSourceFile).readText() + val testFramework = Pytest + val currentPythonModuleOpt = findCurrentPythonModule(directoriesForSysPath, absPathToSourceFile) + val currentPythonModule = when (currentPythonModuleOpt) { + is Success -> { currentPythonModuleOpt.value } + is Fail -> { + logger.error(currentPythonModuleOpt.message) + return + } + } + logger.info("Checking requirements...") + val installer = CliRequirementsInstaller(true, logger) + val requirementsAreInstalled = RequirementsInstaller.checkRequirements( + installer, + pythonPath, + if (testFramework.isInstalled) emptyList() else listOf(testFramework.mainPackage) + ) + if (!requirementsAreInstalled) { + return + } + + val pythonMethodGroups = getPythonMethods().let { if (checkUsvm) it.take(1).map { it.take(1) } else it } + + val sysPathDirectories = directoriesForSysPath.map { it.toAbsolutePath() } .toSet() + val testFile = TestFileInformation(absPathToSourceFile, sourceFileContent, currentPythonModule.dropInitFile()) + + val mypyConfig: MypyConfig + val mypyTime = measureTimeMillis { + logger.info("Loading information about Python types...") + mypyConfig = PythonTestGenerationProcessor.sourceCodeAnalyze( + sysPathDirectories, + pythonPath, + testFile, + ) + } + logger.info { "Mypy time: $mypyTime" } + + addShutdownHook() + + val startTime = System.currentTimeMillis() + val countOfFunctions = pythonMethodGroups.sumOf { it.size } + val timeoutAfterMypy = if (includeMypyAnalysisTime) timeout - mypyTime else timeout + val oneFunctionTimeout = separateTimeout(timeoutAfterMypy, countOfFunctions) + logger.info { "One function timeout: ${oneFunctionTimeout}ms. x${countOfFunctions}" } + pythonMethodGroups.mapIndexed { index, pythonMethods -> + val usedTime = System.currentTimeMillis() - startTime + val countOfTestedFunctions = pythonMethodGroups.take(index).sumOf { it.size } + val expectedTime = countOfTestedFunctions * oneFunctionTimeout + val localOneFunctionTimeout = if (usedTime < expectedTime) { + separateTimeout(timeoutAfterMypy - usedTime, countOfFunctions - countOfTestedFunctions) + } else { + oneFunctionTimeout + } + val localTimeout = pythonMethods.size * localOneFunctionTimeout + logger.info { "Timeout for current group: ${localTimeout}ms" } + + val config = PythonTestGenerationConfig( + pythonPath = pythonPath, + testFileInformation = testFile, + sysPathDirectories = sysPathDirectories, + testedMethods = pythonMethods, + timeout = localTimeout, + timeoutForRun = timeoutForRun, + testFramework = testFramework, + testSourceRootPath = null, + withMinimization = !doNotMinimize, + isCanceled = { shutdown.get() }, + runtimeExceptionTestsBehaviour = RuntimeExceptionTestsBehaviour.valueOf(runtimeExceptionTestsBehaviour), + coverageMeasureMode = PythonCoverageMode.parse(coverageMeasureMode), + sendCoverageContinuously = !doNotSendCoverageContinuously, + coverageOutputFormat = CoverageOutputFormat.Lines, + usvmConfig = UsvmConfig(javaCmd, usvmDirectory), + prohibitedExceptions = if (prohibitedExceptions == listOf("-")) emptyList() else prohibitedExceptions, + checkUsvm = checkUsvm, + doNotGenerateStateAssertions = doNotGenerateStateAssertions, + ) + val processor = SbftCliProcessor(config) + + logger.info("Generating tests...") + val testSets = processor.testGenerate(mypyConfig) + if (testSets.isNotEmpty()) { + val (testCode, imports) = processor.testCodeGenerateSplitImports(testSets) + globalCodeCollection.add(testCode) + globalImportCollection.addAll(imports) + } + } + saveTests() + removeShutdownHook() + exitProcess(0) + } +} diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/Utils.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/Utils.kt new file mode 100644 index 0000000000..661eec5332 --- /dev/null +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/Utils.kt @@ -0,0 +1,20 @@ +package org.utbot.cli.language.python.sbft + +import org.utbot.python.framework.codegen.model.PythonImport +import org.utbot.python.framework.codegen.model.PythonSysPathImport +import java.lang.StringBuilder + +fun renderPythonImport(pythonImport: PythonImport) : String { + val importBuilder = StringBuilder() + if (pythonImport is PythonSysPathImport) { + importBuilder.append("sys.path.append(r'${pythonImport.sysPath}')") + } else if (pythonImport.moduleName == null) { + importBuilder.append("import ${pythonImport.importName}") + } else { + importBuilder.append("from ${pythonImport.moduleName} import ${pythonImport.importName}") + } + if (pythonImport.alias != null) { + importBuilder.append(" as ${pythonImport.alias}") + } + return importBuilder.toString() +} diff --git a/utbot-intellij-python/build.gradle.kts b/utbot-intellij-python/build.gradle.kts index 531d6e75c2..725f92782b 100644 --- a/utbot-intellij-python/build.gradle.kts +++ b/utbot-intellij-python/build.gradle.kts @@ -1,5 +1,7 @@ val semVer: String? by rootProject val kotlinLoggingVersion: String? by rootProject +val apacheCommonsTextVersion: String? by rootProject +val jacksonVersion: String? by rootProject // === IDE settings === val projectType: String by rootProject @@ -122,4 +124,4 @@ intellij { version.set(ideVersion) type.set(ideType) -} \ No newline at end of file +} diff --git a/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/python/PythonDialogProcessor.kt b/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/python/PythonDialogProcessor.kt index a90e62650e..a8dca566b6 100644 --- a/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/python/PythonDialogProcessor.kt +++ b/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/python/PythonDialogProcessor.kt @@ -29,6 +29,7 @@ import org.utbot.common.PathUtil.toPath import org.utbot.framework.plugin.api.util.LockFile import org.utbot.intellij.plugin.settings.Settings import org.utbot.intellij.plugin.ui.utils.showErrorDialogLater +import org.utbot.python.PyDecorator import org.utbot.python.PythonMethodHeader import org.utbot.python.PythonTestGenerationConfig import org.utbot.python.utils.RequirementsInstaller @@ -187,10 +188,17 @@ object PythonDialogProcessor { val moduleFilename = it.containingFile.virtualFile?.canonicalPath ?: "" val containingClassId = it.containingClass?.qualifiedName?.let{ cls -> PythonClassId(cls) } PythonMethodHeader( - functionName, - moduleFilename, - containingClassId, - ) + functionName, + moduleFilename, + containingClassId, + it.decoratorList?.decorators?.mapNotNull { decorator -> + decorator.name?.let { name -> + PyDecorator.decoratorByName( + name + ) + } + } ?: emptyList() + ) } .toSet() .toList() @@ -296,7 +304,7 @@ object PythonDialogProcessor { localUpdateIndicator(ProgressRange.ANALYZE, "Analyze module ${model.currentPythonModule}", 0.5) - val (mypyStorage, _) = processor.sourceCodeAnalyze() + val mypyConfig = processor.sourceCodeAnalyze() localUpdateIndicator(ProgressRange.ANALYZE, "Analyze module ${model.currentPythonModule}", 1.0) @@ -309,7 +317,7 @@ object PythonDialogProcessor { model.timeout, ) try { - val testSets = processor.testGenerate(mypyStorage) + val testSets = processor.testGenerate(mypyConfig) timerHandler.cancel(true) if (testSets.isEmpty()) return@forEachIndexed @@ -321,7 +329,7 @@ object PythonDialogProcessor { logger.info( "Finished test generation for the following functions: ${ - testSets.joinToString { it.method.name } + testSets.map { it.method.name }.toSet().joinToString() }" ) } finally { @@ -426,4 +434,4 @@ fun getDirectoriesForSysPath( importStringPath ) }.executeSynchronously() ?: error("Cannot collect sys path directories") -} \ No newline at end of file +} diff --git a/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/python/Utils.kt b/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/python/Utils.kt index 783126ffcc..21fa5b9969 100644 --- a/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/python/Utils.kt +++ b/utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/python/Utils.kt @@ -52,10 +52,12 @@ fun checkModuleIsInstalled(pythonPath: String, moduleName: String): Boolean { return RequirementsUtils.requirementsAreInstalled(pythonPath, listOf(moduleName)) } -fun fineFunction(function: PyFunction): Boolean = - !listOf("__init__", "__new__").contains(function.name) && - function.decoratorList?.decorators?.isNotEmpty() != true // TODO: add processing of simple decorators - //(function.parent !is PyDecorator || (function.parent as PyDecorator).isBuiltin) +fun fineFunction(function: PyFunction): Boolean { + val hasNotConstructorName = !listOf("__init__", "__new__").contains(function.name) + val decoratorNames = function.decoratorList?.decorators?.mapNotNull { it?.qualifiedName } + val knownDecorators = decoratorNames?.all { it.toString() in listOf("staticmethod") } ?: true + return hasNotConstructorName && knownDecorators +} fun fineClass(pyClass: PyClass): Boolean = getAncestors(pyClass).dropLast(1).all { it !is PyClass && it !is PyFunction } && diff --git a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt index 6b268a71d4..4474e95217 100644 --- a/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt +++ b/utbot-intellij/src/main/kotlin/org/utbot/intellij/plugin/settings/SettingsWindow.kt @@ -64,7 +64,7 @@ class SettingsWindow(val project: Project) { enableSummarizationGenerationCheckBox.isSelected = false } } - } + }.bottomGap(BottomGap.MEDIUM) row("Overflow detection:") { createCombo(TreatOverflowAsError::class, TreatOverflowAsError.values()) diff --git a/utbot-python-executor/src/main/python/utbot_executor/README.md b/utbot-python-executor/src/main/python/utbot_executor/README.md index 05197bc444..38a53d7bcc 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/README.md +++ b/utbot-python-executor/src/main/python/utbot_executor/README.md @@ -29,6 +29,7 @@ $ python -m utbot_executor [ ["] readme = "README.md" diff --git a/utbot-python-executor/src/main/python/utbot_executor/tests/pytest.ini b/utbot-python-executor/src/main/python/utbot_executor/tests/pytest.ini deleted file mode 100644 index d7d93225a4..0000000000 --- a/utbot-python-executor/src/main/python/utbot_executor/tests/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -python_files = test_*.py *_test.py *_tests.py \ No newline at end of file diff --git a/utbot-python-executor/src/main/python/utbot_executor/tests/test_deep_serialization.py b/utbot-python-executor/src/main/python/utbot_executor/tests/test_deep_serialization.py index 58f991562d..154af2809f 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/tests/test_deep_serialization.py +++ b/utbot-python-executor/src/main/python/utbot_executor/tests/test_deep_serialization.py @@ -3,6 +3,7 @@ import datetime import importlib.metadata import json +import pickle import re import sys import typing @@ -28,6 +29,11 @@ def template_test_assert(obj: typing.Any, imports: typing.List[str]): assert obj == get_deserialized_obj(obj, imports) +def template_test_assert_for_generators(obj: typing.Any, imports: typing.List[str]): + items = list(obj) + assert items == list(get_deserialized_obj(iter(items), imports).content) + + @pytest.mark.parametrize( "obj", [ @@ -47,6 +53,10 @@ def template_test_assert(obj: typing.Any, imports: typing.List[str]): ({},), ((1, 2, 3),), (tuple(),), + (pickle.dumps(((2, [1, 2]), {})),), + ("123\n \t ",), + (b"123\n \t ",), + ("\n 123\n \t \n",), ], ) def test_primitives(obj: typing.Any): @@ -78,12 +88,12 @@ class MyDataClass: "obj,imports", [ ( - MyDataClass(1, "a", [1, 2], {"a": b"c"}), - ["tests.test_deep_serialization"], + MyDataClass(1, "a", [1, 2], {"a": b"c"}), + ["tests.test_deep_serialization"], ), ( - MyDataClass(1, "a--------------\n\t", [], {}), - ["tests.test_deep_serialization"], + MyDataClass(1, "a--------------\n\t", [], {}), + ["tests.test_deep_serialization"], ), ], ) @@ -102,10 +112,10 @@ def __eq__(self, other): if not isinstance(other, MyClass): return False return ( - self.a == other.a - and self.b == other.b - and self.c == other.c - and self.d == other.d + self.a == other.a + and self.b == other.b + and self.c == other.c + and self.d == other.d ) @@ -126,12 +136,12 @@ def __eq__(self, other): "obj,imports", [ ( - MyClass(1, "a", [1, 2], {"a": b"c"}), - ["tests.test_deep_serialization"], + MyClass(1, "a", [1, 2], {"a": b"c"}), + ["tests.test_deep_serialization"], ), ( - MyClass(1, "a--------------\n\t", [], {}), - ["tests.test_deep_serialization"], + MyClass(1, "a--------------\n\t", [], {}), + ["tests.test_deep_serialization"], ), (EmptyClass(), ["tests.test_deep_serialization"]), (EmptyInitClass(), ["tests.test_deep_serialization"]), @@ -154,10 +164,10 @@ def __eq__(self, other): if not isinstance(other, MyClassWithSlots): return False return ( - self.a == other.a - and self.b == other.b - and self.c == other.c - and self.d == other.d + self.a == other.a + and self.b == other.b + and self.c == other.c + and self.d == other.d ) def __str__(self): @@ -172,12 +182,12 @@ def __setstate__(self, state): "obj,imports", [ ( - MyClassWithSlots(1, "a", [1, 2], {"a": b"c"}), - ["tests.test_deep_serialization", "copyreg"], + MyClassWithSlots(1, "a", [1, 2], {"a": b"c"}), + ["tests.test_deep_serialization", "copyreg"], ), ( - MyClassWithSlots(1, "a--------------\n\t", [], {}), - ["tests.test_deep_serialization", "copyreg"], + MyClassWithSlots(1, "a--------------\n\t", [], {}), + ["tests.test_deep_serialization", "copyreg"], ), ], ) @@ -185,6 +195,36 @@ def test_classes_with_slots(obj: typing.Any, imports: typing.List[str]): template_test_assert(obj, imports) +def square_iter(x: int): + for i in range(x): + yield i**2 + + +@pytest.mark.parametrize( + "obj,imports", + [ + ( + range(10), + ["tests.test_deep_serialization", "copyreg"], + ), + ( + map(int, "1 1 2 3 4 12 1 239".split()), + ["tests.test_deep_serialization", "copyreg"], + ), + ( + square_iter(5), + ["tests.test_deep_serialization", "copyreg"], + ), + ( + iter([1, 2, 5]), + ["tests.test_deep_serialization", "copyreg"], + ), + ], +) +def test_base_generators(obj: typing.Any, imports: typing.List[str]): + template_test_assert_for_generators(obj, imports) + + def test_comparable(): obj = EmptyClass() serialized_obj_ids, _, serialized_memory_dump = serialize_objects_dump([obj], True) @@ -193,7 +233,7 @@ def test_comparable(): def test_complex(): - obj = complex(real=float('-inf'), imag=float('nan')) + obj = complex(real=float("-inf"), imag=float("nan")) serialized_obj_ids, _, serialized_memory_dump = serialize_objects_dump([obj], True) memory_dump = json_converter.deserialize_memory_objects(serialized_memory_dump) assert not memory_dump.objects[serialized_obj_ids[0]].comparable @@ -204,7 +244,7 @@ class A: def __init__(self, c): self.c = c - obj = A(complex(real=float('-inf'), imag=float('nan'))) + obj = A(complex(real=float("-inf"), imag=float("nan"))) serialized_obj_ids, _, serialized_memory_dump = serialize_objects_dump([obj], True) memory_dump = json_converter.deserialize_memory_objects(serialized_memory_dump) deserialized_obj = memory_dump.objects[serialized_obj_ids[0]] @@ -334,16 +374,16 @@ def test_recursive_object(): "obj,imports", [ ( - collections.Counter("abcababa"), - ["tests.test_deep_serialization", "collections"], + collections.Counter("abcababa"), + ["tests.test_deep_serialization", "collections"], ), ( - collections.UserDict({1: "a"}), - ["tests.test_deep_serialization", "collections"], + collections.UserDict({1: "a"}), + ["tests.test_deep_serialization", "collections"], ), ( - collections.deque([1, 2, 3]), - ["tests.test_deep_serialization", "collections"], + collections.deque([1, 2, 3]), + ["tests.test_deep_serialization", "collections"], ), ], ) @@ -374,20 +414,20 @@ def test_strategy(obj: typing.Any, strategy: str): [ (re.compile(r"\d+jflsf"), ["tests.test_deep_serialization", "re"]), ( - collections.abc.KeysView, - ["tests.test_deep_serialization", "collections"], + collections.abc.KeysView, + ["tests.test_deep_serialization", "collections"], ), ( - collections.abc.KeysView({}), - [ - "tests.test_deep_serialization", - "collections", - "collections.abc", - ], + collections.abc.KeysView({}), + [ + "tests.test_deep_serialization", + "collections", + "collections.abc", + ], ), ( - importlib.metadata.SelectableGroups([["1", "2"]]), - ["tests.test_deep_serialization", "importlib.metadata"], + importlib.metadata.SelectableGroups([["1", "2"]]), + ["tests.test_deep_serialization", "importlib.metadata"], ), ], ) @@ -401,7 +441,7 @@ def test_corner_cases(obj: typing.Any, imports: typing.List[str]): @pytest.mark.skipif( sys.version_info.major <= 3 and sys.version_info.minor < 11, reason="typing.TypeVarTuple (PEP 646) has been added in Python 3.11", - ) +) def test_type_var_tuple(): globals()["T2"] = typing.TypeVarTuple("T2") obj = typing.TypeVarTuple("T2") diff --git a/utbot-python-executor/src/main/python/utbot_executor/tests/test_executor.py b/utbot-python-executor/src/main/python/utbot_executor/tests/test_executor.py new file mode 100644 index 0000000000..c4f9ec57b8 --- /dev/null +++ b/utbot-python-executor/src/main/python/utbot_executor/tests/test_executor.py @@ -0,0 +1,53 @@ +from utbot_executor.deep_serialization.deep_serialization import serialize_objects_dump +from utbot_executor.deep_serialization.memory_objects import ( + ReprMemoryObject, + ReduceMemoryObject, +) + + +def test_serialize_state(): + args = ["\n 123 \n"] + kwargs = {} + result = None + + _, state, serialized_state = serialize_objects_dump(args + list(kwargs) + [result]) + + serialized_arg = list(state.objects.values())[0] + assert isinstance(serialized_arg, ReprMemoryObject) + assert serialized_arg.value == "'\\n 123 \\n'" + + +def test_serialize_state_1(): + args = ["0\n 123 \n"] + kwargs = {} + result = None + + _, state, serialized_state = serialize_objects_dump(args + list(kwargs) + [result]) + + serialized_arg = list(state.objects.values())[0] + assert isinstance(serialized_arg, ReprMemoryObject) + assert serialized_arg.value == "'0\\n 123 \\n'" + + +def test_serialize_state_2(): + args = ["\\\n Adds new strings"] + kwargs = {} + result = None + + _, state, serialized_state = serialize_objects_dump(args + list(kwargs) + [result]) + + serialized_arg = list(state.objects.values())[0] + assert isinstance(serialized_arg, ReprMemoryObject) + assert serialized_arg.value == "'\\\\\\n Adds new strings'" + + +class A: + class B: + def __init__(self, x): + self.x = x + + +def test_serialize_inner_class(): + b = A.B(1) + serialized_b = ReduceMemoryObject(b) + assert "A.B" in serialized_b.constructor.qualname diff --git a/utbot-python-executor/src/main/python/utbot_executor/tests/test_python_executor.py b/utbot-python-executor/src/main/python/utbot_executor/tests/test_python_executor.py new file mode 100644 index 0000000000..1199de52e5 --- /dev/null +++ b/utbot-python-executor/src/main/python/utbot_executor/tests/test_python_executor.py @@ -0,0 +1,33 @@ +from utbot_executor.config import CoverageConfig, HostConfig +from utbot_executor.executor import PythonExecutor +import random + +from utbot_executor.parser import ( + ExecutionRequest, + parse_request, + ExecutionSuccessResponse, +) +from utbot_executor.utils import TraceMode + +random.seed(239) + + +def _generate_host_config() -> HostConfig: + return HostConfig("localhost", random.randint(10**5, 10**6)) + + +def _generate_coverage_config() -> CoverageConfig: + return CoverageConfig(_generate_host_config(), TraceMode.Instructions, True) + + +def _read_request() -> ExecutionRequest: + with open("example_input.json", "r") as fin: + text = "\n".join(fin.readlines()) + return parse_request(text) + + +def test_python_executor(): + executor = PythonExecutor(_generate_coverage_config(), False) + request = _read_request() + resource = executor.run_function(request) + assert isinstance(resource.status, ExecutionSuccessResponse) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py index c9356cdcdb..752a1df3cf 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/__main__.py @@ -1,12 +1,13 @@ import argparse import logging +from utbot_executor.config import Config, HostConfig, CoverageConfig, LoggingConfig from utbot_executor.listener import PythonExecuteServer from utbot_executor.utils import TraceMode -def main(hostname: str, port: int, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode, send_coverage: bool): - server = PythonExecuteServer(hostname, port, coverage_hostname, coverage_port, trace_mode, send_coverage) +def main(executor_config: Config): + server = PythonExecuteServer(executor_config) server.run() @@ -28,9 +29,8 @@ def main(hostname: str, port: int, coverage_hostname: str, coverage_port: int, t parser.add_argument( "--coverage_type", choices=["lines", "instructions"], default="instructions" ) - parser.add_argument( - "--send_coverage", action=argparse.BooleanOptionalAction - ) + parser.add_argument("--send_coverage", action=argparse.BooleanOptionalAction) + parser.add_argument("--generate_state_assertions", action=argparse.BooleanOptionalAction) args = parser.parse_args() loglevel = { @@ -45,6 +45,17 @@ def main(hostname: str, port: int, coverage_hostname: str, coverage_port: int, t datefmt="%m/%d/%Y %H:%M:%S", level=loglevel, ) - trace_mode = TraceMode.Lines if args.coverage_type == "lines" else TraceMode.Instructions - send_coverage = args.send_coverage - main(args.hostname, args.port, args.coverage_hostname, args.coverage_port, trace_mode, send_coverage) + trace_mode = ( + TraceMode.Lines if args.coverage_type == "lines" else TraceMode.Instructions + ) + + config = Config( + server=HostConfig(args.hostname, args.port), + coverage=CoverageConfig( + HostConfig(args.coverage_hostname, args.coverage_port), trace_mode, args.send_coverage + ), + logging=LoggingConfig(args.logfile, loglevel), + state_assertions=args.generate_state_assertions + ) + + main(config) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/config.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/config.py new file mode 100644 index 0000000000..f4d8ff3d2a --- /dev/null +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/config.py @@ -0,0 +1,30 @@ +import dataclasses + +from utbot_executor.utils import TraceMode + + +@dataclasses.dataclass +class HostConfig: + hostname: str + port: int + + +@dataclasses.dataclass +class CoverageConfig: + server: HostConfig + trace_mode: TraceMode + send_coverage: bool + + +@dataclasses.dataclass +class LoggingConfig: + logfile: str | None + loglevel: int + + +@dataclasses.dataclass +class Config: + server: HostConfig + coverage: CoverageConfig + logging: LoggingConfig + state_assertions: bool diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/iterator_wrapper.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/iterator_wrapper.py new file mode 100644 index 0000000000..7bb7161521 --- /dev/null +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/iterator_wrapper.py @@ -0,0 +1,74 @@ +from __future__ import annotations +import itertools +import typing + +T = typing.TypeVar("T") + + +class IteratorWrapper: + iterator: typing.Iterator[T] + content: typing.List[T] + + index = -1 + + MAX_SIZE = 1_000 + + def __init__(self, iterator: typing.Iterator[T]) -> None: + self.iterator, iter_copy = itertools.tee(iterator) + self.content = [] + self.stop_exception = StopIteration + + pair_iter = zip(iter_copy, range(IteratorWrapper.MAX_SIZE)) + while True: + try: + it = next(pair_iter) + self.content.append(it[0]) + except Exception as exc: + self.stop_exception = exc + break + + def build_from_list(self, content: typing.List[T], stop_exception: Exception = StopIteration) -> None: + self.content = content + self.iterator = iter(content) + self.stop_exception = stop_exception + + @staticmethod + def from_list(content: typing.List[T], stop_iteration: Exception = StopIteration) -> IteratorWrapper: + obj = IteratorWrapper.__new__(IteratorWrapper) + obj.build_from_list(content, stop_iteration) + return obj + + def __iter__(self): + self.index = -1 + return self + + def __next__(self): + if self.index + 1 >= len(self.content): + raise StopIteration + self.index += 1 + return self.content[self.index] + + def __eq__(self, other) -> bool: + if isinstance(other, IteratorWrapper): + return self.content == other.content + return False + + def __str__(self) -> str: + return f"IteratorWrapper({self.content})" + + def __getstate__(self) -> typing.Dict[str, typing.Any]: + return {"content": self.content, "stop_exception": self.stop_exception} + + def __setstate__(self, state) -> None: + self.build_from_list(state["content"], state["stop_exception"]) + + +if __name__ == "__main__": + wrapper = IteratorWrapper(iter([1, 2, 3])) + for i in wrapper: + print(i) + # copy.deepcopy(wrapper) + # print(wrapper.__reduce__()) + # a = pickle.dumps(wrapper) + # print(a) + # print(str(pickle.loads(a))) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/json_converter.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/json_converter.py index 753fd8a060..ca27513956 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/json_converter.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/json_converter.py @@ -3,13 +3,15 @@ import json import sys from typing import Dict, Iterable, Union + +from utbot_executor.deep_serialization.iterator_wrapper import IteratorWrapper from utbot_executor.deep_serialization.memory_objects import ( MemoryObject, ReprMemoryObject, ListMemoryObject, DictMemoryObject, ReduceMemoryObject, - MemoryDump, + MemoryDump, IteratorMemoryObject, ) from utbot_executor.deep_serialization.utils import PythonId, TypeInfo @@ -27,6 +29,9 @@ def default(self, o): base_json["value"] = o.value elif isinstance(o, (ListMemoryObject, DictMemoryObject)): base_json["items"] = o.items + elif isinstance(o, IteratorMemoryObject): + base_json["items"] = o.items + base_json["exception"] = o.exception elif isinstance(o, ReduceMemoryObject): base_json["constructor"] = o.constructor base_json["args"] = o.args @@ -51,7 +56,7 @@ def default(self, o): return json.JSONEncoder.default(self, o) -def as_repr_object(dct: Dict) -> Union[MemoryObject, Dict]: +def as_reduce_object(dct: Dict) -> Union[MemoryObject, Dict]: if "strategy" in dct: obj: MemoryObject if dct["strategy"] == "repr": @@ -78,6 +83,15 @@ def as_repr_object(dct: Dict) -> Union[MemoryObject, Dict]: ) obj.comparable = dct["comparable"] return obj + if dct["strategy"] == "iterator": + obj = IteratorMemoryObject.__new__(IteratorMemoryObject) + obj.items = dct["items"] + obj.exception = dct["exception"] + obj.typeinfo = TypeInfo( + kind=dct["typeinfo"]["kind"], module=dct["typeinfo"]["module"] + ) + obj.comparable = dct["comparable"] + return obj if dct["strategy"] == "reduce": obj = ReduceMemoryObject.__new__(ReduceMemoryObject) obj.constructor = TypeInfo( @@ -97,7 +111,7 @@ def as_repr_object(dct: Dict) -> Union[MemoryObject, Dict]: def deserialize_memory_objects(memory_dump: str) -> MemoryDump: - parsed_data = json.loads(memory_dump, object_hook=as_repr_object) + parsed_data = json.loads(memory_dump, object_hook=as_reduce_object) return MemoryDump(parsed_data["objects"]) @@ -110,15 +124,22 @@ def __init__(self, memory_dump: MemoryDump): def reload_id(self) -> MemoryDump: new_memory_objects: Dict[PythonId, MemoryObject] = {} for id_, obj in self.memory_dump.objects.items(): - new_memory_object = copy.deepcopy(obj) - read_id = self.dump_id_to_real_id[id_] - new_memory_object.obj = self.memory[read_id] + try: + new_memory_object = copy.deepcopy(obj) + except TypeError as _: + new_memory_object = self + real_id = self.dump_id_to_real_id[id_] + new_memory_object.obj = self.memory[real_id] if isinstance(new_memory_object, ReprMemoryObject): pass elif isinstance(new_memory_object, ListMemoryObject): new_memory_object.items = [ self.dump_id_to_real_id[id_] for id_ in new_memory_object.items ] + elif isinstance(new_memory_object, IteratorMemoryObject): + new_memory_object.items = [ + self.dump_id_to_real_id[id_] for id_ in new_memory_object.items + ] elif isinstance(new_memory_object, DictMemoryObject): new_memory_object.items = { self.dump_id_to_real_id[id_key]: self.dump_id_to_real_id[id_value] @@ -135,7 +156,7 @@ def reload_id(self) -> MemoryDump: new_memory_object.dictitems = self.dump_id_to_real_id[ new_memory_object.dictitems ] - new_memory_objects[self.dump_id_to_real_id[id_]] = new_memory_object + new_memory_objects[real_id] = new_memory_object return MemoryDump(new_memory_objects) @staticmethod @@ -184,6 +205,14 @@ def load_object(self, python_id: PythonId) -> object: for key, value in dump_object.items.items(): real_object[self.load_object(key)] = self.load_object(value) + elif isinstance(dump_object, IteratorMemoryObject): + real_object = IteratorWrapper.from_list( + [self.load_object(item) for item in dump_object.items], + eval(dump_object.exception.qualname) + ) + id_ = PythonId(str(id(real_object))) + self.dump_id_to_real_id[python_id] = id_ + self.memory[id_] = real_object elif isinstance(dump_object, ReduceMemoryObject): constructor = eval(dump_object.constructor.qualname) args = self.load_object(dump_object.args) @@ -200,20 +229,14 @@ def load_object(self, python_id: PythonId) -> object: state = self.load_object(dump_object.state) if isinstance(state, dict): for field, value in state.items(): - try: - setattr(real_object, field, value) - except AttributeError: - pass + setattr(real_object, field, value) elif hasattr(real_object, "__setstate__"): real_object.__setstate__(state) if isinstance(state, tuple) and len(state) == 2: _, slotstate = state if slotstate: for key, value in slotstate.items(): - try: - setattr(real_object, key, value) - except AttributeError: - pass + setattr(real_object, key, value) listitems = self.load_object(dump_object.listitems) if isinstance(listitems, Iterable): diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/memory_objects.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/memory_objects.py index 5acc0703a5..d54e557bdd 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/memory_objects.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/memory_objects.py @@ -1,16 +1,15 @@ from __future__ import annotations -import copyreg import inspect import logging +import pickle import re import sys import typing from itertools import zip_longest -import pickle from typing import Any, Callable, Dict, List, Optional, Set, Type, Iterable - from utbot_executor.deep_serialization.config import PICKLE_PROTO +from utbot_executor.deep_serialization.iterator_wrapper import IteratorWrapper from utbot_executor.deep_serialization.utils import ( PythonId, get_kind, @@ -32,11 +31,13 @@ class MemoryObject: is_draft: bool deserialized_obj: object obj: object + id_: PythonId | None = None def __init__(self, obj: object) -> None: self.is_draft = True self.typeinfo = get_kind(obj) self.obj = obj + self.id_ = PythonId(str(id(self.obj))) def _initialize( self, deserialized_obj: object = None, comparable: bool = True @@ -49,7 +50,9 @@ def initialize(self) -> None: self._initialize() def id_value(self) -> str: - return str(id(self.obj)) + if self.id_ is not None: + return self.id_ + return PythonId(str(id(self.obj))) def __repr__(self) -> str: if hasattr(self, "obj"): @@ -140,8 +143,8 @@ def initialize(self) -> None: deserialized_obj = self.deserialized_obj equals_len = len(self.obj) == len(deserialized_obj) comparable = equals_len and all( - serializer.get_by_id(value_id).comparable - for value_id in self.items.values() + serializer.get_by_id(value_id).comparable and serializer.get_by_id(key_id).comparable + for key_id, value_id in self.items.items() ) super()._initialize(deserialized_obj, comparable) @@ -152,6 +155,47 @@ def __repr__(self) -> str: return f"{self.typeinfo.kind}{self.items}" +class IteratorMemoryObject(MemoryObject): + strategy: str = "iterator" + items: List[PythonId] + exception: TypeInfo + + MAX_SIZE = 1_000 + + def __init__(self, iterator_object: object) -> None: + self.items = [] + if not isinstance(iterator_object, IteratorWrapper): + iterator_object = IteratorWrapper(iterator_object) + super().__init__(iterator_object) + + def initialize(self) -> None: + self.obj: IteratorWrapper + serializer = PythonSerializer() + self.comparable = False + + for item in self.obj.content: + elem_id = serializer.write_object_to_memory(item) + self.items.append(elem_id) + self.exception = get_kind(self.obj.stop_exception) + + items = [ + serializer.get_by_id(elem_id) + for elem_id in self.items + ] + comparable = all( + item.comparable + for item in items + ) + + deserialized_obj = IteratorWrapper.from_list(items) + super()._initialize(deserialized_obj, comparable) + + def __repr__(self) -> str: + if hasattr(self, "obj"): + return str(self.obj) + return f"{self.typeinfo.kind}{self.items}" + + class ReduceMemoryObject(MemoryObject): strategy: str = "reduce" constructor: TypeInfo @@ -307,20 +351,14 @@ def initialize(self) -> None: state = serializer[self.state] if isinstance(state, dict): for key, value in state.items(): - try: - setattr(deserialized_obj, key, value) - except AttributeError: - pass + setattr(deserialized_obj, key, value) elif hasattr(deserialized_obj, "__setstate__"): deserialized_obj.__setstate__(state) elif isinstance(state, tuple) and len(state) == 2: _, slotstate = state if slotstate: for key, value in slotstate.items(): - try: - setattr(deserialized_obj, key, value) - except AttributeError: - pass + setattr(deserialized_obj, key, value) items = serializer[self.listitems] if isinstance(items, Iterable): @@ -332,7 +370,10 @@ def initialize(self) -> None: for key, value in dictitems.items(): deserialized_obj[key] = value - comparable = self.obj == deserialized_obj + try: + comparable = self.obj == deserialized_obj + except: + comparable = False super()._initialize(deserialized_obj, comparable) @@ -359,6 +400,14 @@ def get_serializer(obj: object) -> Optional[Type[MemoryObject]]: return None +class IteratorMemoryObjectProvider(MemoryObjectProvider): + @staticmethod + def get_serializer(obj: object) -> Optional[Type[MemoryObject]]: + if isinstance(obj, (typing.Iterator, IteratorWrapper)): + return IteratorMemoryObject + return None + + class ReduceMemoryObjectProvider(MemoryObjectProvider): @staticmethod def get_serializer(obj: object) -> Optional[Type[MemoryObject]]: @@ -402,6 +451,7 @@ class PythonSerializer: providers: List[MemoryObjectProvider] = [ ListMemoryObjectProvider, DictMemoryObjectProvider, + IteratorMemoryObjectProvider, ReduceMemoryObjectProvider, ReprMemoryObjectProvider, ReduceExMemoryObjectProvider, diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/utils.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/utils.py index da06aaeb0a..2d268c769d 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/utils.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/deep_serialization/utils.py @@ -1,10 +1,10 @@ from __future__ import annotations + import dataclasses import importlib import logging import pickle from typing import NewType - from utbot_executor.deep_serialization.config import PICKLE_PROTO PythonId = NewType("PythonId", str) @@ -89,7 +89,7 @@ def get_constructor_info(constructor: object, obj: object) -> TypeInfo: result = TypeInfo(constructor.__module__, constructor.__qualname__) if result.kind == "object.__new__" and obj.__new__.__module__ is None: - result = TypeInfo(obj.__module__, f"{obj.__class__.__name__}.__new__") + result = TypeInfo(obj.__module__, f"{obj.__class__.__qualname__}.__new__") return result diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/example/example.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/example/example.py index c16550ba84..7c233d697d 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/example/example.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/example/example.py @@ -17,7 +17,7 @@ def test_execution(): '/home/vyacheslav/Projects/utbot_executor/utbot_executor/tests/my_func.py', '0x1' ) - response = executor.run_function(request) + response = executor.run_reduce_function(request) assert isinstance(response, ExecutionSuccessResponse) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py index 5bedef3896..74c167d923 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/executor.py @@ -4,30 +4,48 @@ import inspect import logging import pathlib +import pickle import sys import traceback import types -from typing import Any, Callable, Dict, Iterable, List, Tuple +from typing import Any, Dict, Iterable, List, Tuple -from utbot_executor.deep_serialization.deep_serialization import serialize_memory_dump, \ - serialize_objects_dump -from utbot_executor.deep_serialization.json_converter import DumpLoader, deserialize_memory_objects -from utbot_executor.deep_serialization.memory_objects import MemoryDump, PythonSerializer +from utbot_executor.config import CoverageConfig +from utbot_executor.deep_serialization.deep_serialization import ( + serialize_memory_dump, + serialize_objects_dump, +) +from utbot_executor.deep_serialization.json_converter import ( + DumpLoader, + deserialize_memory_objects, +) +from utbot_executor.deep_serialization.memory_objects import ( + MemoryDump, + PythonSerializer, +) from utbot_executor.deep_serialization.utils import PythonId, getattr_by_path from utbot_executor.memory_compressor import compress_memory -from utbot_executor.parser import ExecutionRequest, ExecutionResponse, ExecutionFailResponse, ExecutionSuccessResponse +from utbot_executor.parser import ( + ExecutionRequest, + ExecutionResponse, + ExecutionFailResponse, + ExecutionSuccessResponse, + MemoryMode, +) from utbot_executor.ut_tracer import UtTracer, UtCoverageSender from utbot_executor.utils import ( suppress_stdout as __suppress_stdout, get_instructions, filter_instructions, - TraceMode, UtInstruction, + UtInstruction, ) -__all__ = ['PythonExecutor'] +__all__ = ["PythonExecutor"] -def _update_states(init_memory_dump: MemoryDump, before_memory_dump: MemoryDump) -> MemoryDump: +def _update_states( + init_memory_dump: MemoryDump, before_memory_dump: MemoryDump +) -> MemoryDump: for id_, obj in before_memory_dump.objects.items(): if id_ in init_memory_dump.objects: init_memory_dump.objects[id_].comparable = obj.comparable @@ -45,11 +63,12 @@ def _load_objects(objs: List[Any]) -> MemoryDump: class PythonExecutor: - def __init__(self, coverage_hostname: str, coverage_port: int, trace_mode: TraceMode, send_coverage: bool): - self.coverage_hostname = coverage_hostname - self.coverage_port = coverage_port - self.trace_mode = trace_mode - self.send_coverage = send_coverage + def __init__(self, coverage_config: CoverageConfig, state_assertions: bool): + self.coverage_hostname = coverage_config.server.hostname + self.coverage_port = coverage_config.server.port + self.trace_mode = coverage_config.trace_mode + self.send_coverage = coverage_config.send_coverage + self.state_assertions = state_assertions @staticmethod def add_syspaths(syspaths: Iterable[str]): @@ -60,17 +79,26 @@ def add_syspaths(syspaths: Iterable[str]): @staticmethod def add_imports(imports: Iterable[str]): for module in imports: - for i in range(1, module.count('.') + 2): - submodule_name = '.'.join(module.split('.', maxsplit=i)[:i]) + for i in range(1, module.count(".") + 2): + submodule_name = ".".join(module.split(".", maxsplit=i)[:i]) logging.debug("Submodule #%d: %s", i, submodule_name) if submodule_name not in globals(): try: - globals()[submodule_name] = importlib.import_module(submodule_name) + globals()[submodule_name] = importlib.import_module( + submodule_name + ) except ModuleNotFoundError: logging.warning("Import submodule %s failed", submodule_name) logging.debug("Submodule #%d: OK", i) def run_function(self, request: ExecutionRequest) -> ExecutionResponse: + match request.memory_mode: + case MemoryMode.PICKLE: + return self.run_pickle_function(request) + case MemoryMode.REDUCE: + return self.run_reduce_function(request) + + def run_reduce_function(self, request: ExecutionRequest) -> ExecutionResponse: logging.debug("Prepare to run function `%s`", request.function_name) try: memory_dump = deserialize_memory_objects(request.serialized_memory) @@ -94,18 +122,22 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse: try: function = getattr_by_path( - importlib.import_module(request.function_module), - request.function_name - ) + importlib.import_module(request.function_module), request.function_name + ) if not isinstance(function, types.FunctionType): return ExecutionFailResponse( - "fail", - f"Invalid function path {request.function_module}.{request.function_name}" - ) + "fail", + f"Invalid function path {request.function_module}.{request.function_name}", + ) logging.debug("Function initialized") - args = [loader.load_object(PythonId(arg_id)) for arg_id in request.arguments_ids] + args = [ + loader.load_object(PythonId(arg_id)) for arg_id in request.arguments_ids + ] logging.debug("Arguments: %s", args) - kwargs = {name: loader.load_object(PythonId(kwarg_id)) for name, kwarg_id in request.kwarguments_ids.items()} + kwargs = { + name: loader.load_object(PythonId(kwarg_id)) + for name, kwarg_id in request.kwarguments_ids.items() + } logging.debug("Kwarguments: %s", kwargs) except Exception as _: logging.debug("Error \n%s", traceback.format_exc()) @@ -136,6 +168,75 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse: _coverage_sender, self.trace_mode, ), + state_assertions=self.state_assertions, + ) + except Exception as _: + logging.debug("Error \n%s", traceback.format_exc()) + return ExecutionFailResponse("fail", traceback.format_exc()) + logging.debug("Value have been calculated: %s", value) + return value + + def run_pickle_function(self, request: ExecutionRequest) -> ExecutionResponse: + logging.debug("Prepare to run function `%s`", request.function_name) + try: + logging.debug("Imports: %s", request.imports) + logging.debug("Syspaths: %s", request.syspaths) + self.add_syspaths(request.syspaths) + self.add_imports(request.imports) + except Exception as _: + logging.debug("Error \n%s", traceback.format_exc()) + return ExecutionFailResponse("fail", traceback.format_exc()) + logging.debug("Imports have been added") + + try: + function = getattr_by_path( + importlib.import_module(request.function_module), request.function_name + ) + if not isinstance(function, types.FunctionType): + return ExecutionFailResponse( + "fail", + f"Invalid function path {request.function_module}.{request.function_name}", + ) + logging.debug("Function initialized") + args, kwargs = pickle.loads(eval(request.serialized_memory)) + logging.debug("Arguments: %s", args) + logging.debug("Kwarguments: %s", kwargs) + class_name = request.get_class_name() + if class_name is not None: + real_class = getattr_by_path( + importlib.import_module(request.function_module), class_name + ) + if not isinstance(args[0], real_class): + error_message = f"Invalid self argument \n{type(args[0])} instead of {class_name}" + logging.debug(error_message) + return ExecutionFailResponse("fail", error_message) + + except Exception as _: + logging.debug("Error \n%s", traceback.format_exc()) + return ExecutionFailResponse("fail", traceback.format_exc()) + logging.debug("Arguments have been created") + + try: + _coverage_sender = UtCoverageSender( + request.coverage_id, + self.coverage_hostname, + self.coverage_port, + send_coverage=self.send_coverage, + ) + + value = _run_calculate_function_value( + function, + list(args), + kwargs, + request.filepath, + "", + tracer=UtTracer( + pathlib.Path(request.filepath), + [sys.prefix, sys.exec_prefix], + _coverage_sender, + self.trace_mode, + ), + state_assertions=self.state_assertions, ) except Exception as _: logging.debug("Error \n%s", traceback.format_exc()) @@ -145,10 +246,10 @@ def run_function(self, request: ExecutionRequest) -> ExecutionResponse: def _serialize_state( - args: List[Any], - kwargs: Dict[str, Any], - result: Any = None, - ) -> Tuple[List[PythonId], Dict[str, PythonId], PythonId, MemoryDump, str]: + args: List[Any], + kwargs: Dict[str, Any], + result: Any = None, +) -> Tuple[List[PythonId], Dict[str, PythonId], PythonId, MemoryDump, str]: """Serialize objects from args, kwargs and result. Returns: tuple of args ids, kwargs ids, result id and serialized memory.""" @@ -157,23 +258,24 @@ def _serialize_state( ids, memory, serialized_memory = serialize_objects_dump(all_arguments, True) return ( - ids[:len(args)], - dict(zip(kwargs.keys(), ids[len(args):len(args)+len(kwargs)])), - ids[-1], - copy.deepcopy(memory), - serialized_memory, - ) + ids[: len(args)], + dict(zip(kwargs.keys(), ids[len(args) : len(args) + len(kwargs)])), + ids[-1], + copy.deepcopy(memory), + serialized_memory, + ) def _run_calculate_function_value( - function: types.FunctionType, - args: List[Any], - kwargs: Dict[str, Any], - fullpath: str, - state_init: str, - tracer: UtTracer, - ) -> ExecutionResponse: - """ Calculate function evaluation result. + function: types.FunctionType, + args: List[Any], + kwargs: Dict[str, Any], + fullpath: str, + state_init: str, + tracer: UtTracer, + state_assertions: bool, +) -> ExecutionResponse: + """Calculate function evaluation result. Return serialized data: status, coverage info, object ids and memory.""" @@ -181,43 +283,66 @@ def _run_calculate_function_value( __is_exception = False - _, __start = inspect.getsourcelines(function) - __all_code_stmts = filter_instructions(get_instructions(function), tracer.mode) + __source_lines, __start = inspect.getsourcelines(function) + __end = __start + len(__source_lines) + __all_code_stmts = filter_instructions( + get_instructions(function.__code__), tracer.mode + ) __tracer = tracer - try: - with __suppress_stdout(): - __result = __tracer.runfunc(function, *args, **kwargs) - except Exception as __exception: - __result = __exception - __is_exception = True + with __suppress_stdout(): + try: + __result = __tracer.runfunc( + function, __tracer.DEFAULT_LINE_FILTER, *args, **kwargs + ) + except Exception as __exception: + __result = __exception + __is_exception = True + try: + ( + args_ids, + kwargs_ids, + result_id, + state_after, + serialized_state_after, + ) = __tracer.runfunc( + _serialize_state, (__start, __end), args, kwargs, __result + ) + except Exception as __exception: + _, _, e_traceback = sys.exc_info() + e_filename = e_traceback.tb_frame.f_code.co_filename + if e_filename == fullpath: + __result = __exception + __is_exception = True + logging.debug("Function call finished: %s", __result) logging.debug("Coverage: %s", __tracer.counts) logging.debug("Fullpath: %s", fullpath) - __stmts_with_def = [UtInstruction(__start, 0, True)] + list(__tracer.counts.keys()) + __stmts_with_def = [UtInstruction(__start, 0, True)] + __tracer.instructions __missed_filtered = [x for x in __all_code_stmts if x not in __stmts_with_def] logging.debug("Covered lines: %s", __stmts_with_def) logging.debug("Missed lines: %s", __missed_filtered) __str_statements = [x.serialize() for x in __stmts_with_def] __str_missed_statements = [x.serialize() for x in __missed_filtered] - - args_ids, kwargs_ids, result_id, state_after, serialized_state_after = _serialize_state(args, kwargs, __result) ids = args_ids + list(kwargs_ids.values()) - diff_ids = compress_memory(ids, state_before, state_after) + if (state_assertions or __result is None) and not inspect.isgenerator(__result): + diff_ids = compress_memory(ids, state_before, state_after) + else: + diff_ids = [] return ExecutionSuccessResponse( - status="success", - is_exception=__is_exception, - statements=__str_statements, - missed_statements=__str_missed_statements, - state_init=state_init, - state_before=serialized_state_before, - state_after=serialized_state_after, - diff_ids=diff_ids, - args_ids=args_ids, - kwargs_ids=kwargs_ids, - result_id=result_id, - ) + status="success", + is_exception=__is_exception, + statements=__str_statements, + missed_statements=__str_missed_statements, + state_init=state_init, + state_before=serialized_state_before, + state_after=serialized_state_after, + diff_ids=diff_ids, + args_ids=args_ids, + kwargs_ids=kwargs_ids, + result_id=result_id, + ) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py index a1c91e8c00..282215960a 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/listener.py @@ -2,83 +2,84 @@ import os import socket import traceback - +from utbot_executor.config import Config from utbot_executor.deep_serialization.memory_objects import PythonSerializer -from utbot_executor.parser import parse_request, serialize_response, ExecutionFailResponse from utbot_executor.executor import PythonExecutor -from utbot_executor.utils import TraceMode +from utbot_executor.parser import ( + parse_request, + serialize_response, + ExecutionFailResponse, +) RECV_SIZE = 2**15 class PythonExecuteServer: - def __init__( - self, - hostname: str, - port: int, - coverage_hostname: str, - coverage_port: int, - trace_mode: TraceMode, - send_coverage: bool - ): - logging.info('PythonExecutor is creating...') + def __init__(self, config: Config): + logging.info("PythonExecutor is creating...") self.clientsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.clientsocket.connect((hostname, port)) - self.executor = PythonExecutor(coverage_hostname, coverage_port, trace_mode, send_coverage) + self.clientsocket.connect((config.server.hostname, config.server.port)) + self.executor = PythonExecutor( + config.coverage, + config.state_assertions, + ) + self.config = config def run(self) -> None: - logging.info('PythonExecutor is ready...') + logging.info("PythonExecutor is ready...") try: self.handler() finally: self.clientsocket.close() def handler(self) -> None: - logging.info('Start working...') + logging.info("Start working...") while True: command = self.clientsocket.recv(4) - if command == b'STOP': + if command == b"STOP": break - if command == b'DATA': + if command == b"DATA": message_size = int(self.clientsocket.recv(16).decode()) - logging.debug('Got message size: %d bytes', message_size) + logging.debug("Got message size: %d bytes", message_size) message_body = bytearray() while len(message_body) < message_size: message = self.clientsocket.recv( - min(RECV_SIZE, message_size - len(message_body)) - ) + min(RECV_SIZE, message_size - len(message_body)) + ) message_body += message - logging.debug('Message: %s, size: %d', message, len(message)) + logging.debug("Message: %s, size: %d", message, len(message)) logging.debug( - 'Update content, current size: %d / %d bytes', + "Update content, current size: %d / %d bytes", len(message_body), message_size, ) try: request = parse_request(message_body.decode()) - logging.debug('Parsed request: %s', request) + logging.debug("Parsed request: %s", request) response = self.executor.run_function(request) except Exception as ex: - logging.debug('Exception: %s', traceback.format_exc()) - response = ExecutionFailResponse('fail', traceback.format_exc()) + logging.debug("Exception: %s", traceback.format_exc()) + response = ExecutionFailResponse("fail", traceback.format_exc()) - logging.debug('Response: %s', response) + logging.debug("Response: %s", response) try: serialized_response = serialize_response(response) except Exception as ex: - serialized_response = serialize_response(ExecutionFailResponse('fail', '')) + serialized_response = serialize_response( + ExecutionFailResponse("fail", "") + ) finally: PythonSerializer().clear() - logging.debug('Serialized response: %s', serialized_response) + logging.debug("Serialized response: %s", serialized_response) bytes_data = serialized_response.encode() - logging.debug('Encoded response: %s', bytes_data) + logging.debug("Encoded response: %s", bytes_data) response_size = str(len(bytes_data)) self.clientsocket.send((response_size + os.linesep).encode()) @@ -86,5 +87,5 @@ def handler(self) -> None: while len(bytes_data) > sent_size: sent_size += self.clientsocket.send(bytes_data[sent_size:]) - logging.debug('Sent all data') - logging.info('All done...') + logging.debug("Sent all data") + logging.info("All done...") diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/memory_compressor.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/memory_compressor.py index feb6ea869b..8d3ccb5fe7 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/memory_compressor.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/memory_compressor.py @@ -12,6 +12,9 @@ def compress_memory( diff_ids: typing.List[PythonId] = [] for id_ in ids: if id_ in state_before.objects and id_ in state_after.objects: - if state_before.objects[id_].obj != state_after.objects[id_].obj: - diff_ids.append(id_) + try: + if state_before.objects[id_].obj != state_after.objects[id_].obj: + diff_ids.append(id_) + except AttributeError as _: + pass return diff_ids diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py index d247c28290..cb54c0e657 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/parser.py @@ -1,8 +1,14 @@ import dataclasses +import enum import json from typing import Dict, List, Union, Tuple +class MemoryMode(enum.StrEnum): + PICKLE = "PICKLE" + REDUCE = "REDUCE" + + @dataclasses.dataclass class ExecutionRequest: function_name: str @@ -12,9 +18,15 @@ class ExecutionRequest: arguments_ids: List[str] kwarguments_ids: Dict[str, str] serialized_memory: str + memory_mode: MemoryMode filepath: str coverage_id: str + def get_class_name(self) -> str | None: + if "." in self.function_name: + return self.function_name.rsplit(".", 1)[0] + return None + class ExecutionResponse: status: str @@ -41,7 +53,7 @@ class ExecutionFailResponse(ExecutionResponse): exception: str -def as_execution_result(dct: Dict) -> Union[ExecutionRequest, Dict]: +def as_execution_request(dct: Dict) -> Union[ExecutionRequest, Dict]: if set(dct.keys()) == { 'functionName', 'functionModule', @@ -50,6 +62,7 @@ def as_execution_result(dct: Dict) -> Union[ExecutionRequest, Dict]: 'argumentsIds', 'kwargumentsIds', 'serializedMemory', + 'memoryMode', 'filepath', 'coverageId', }: @@ -61,6 +74,7 @@ def as_execution_result(dct: Dict) -> Union[ExecutionRequest, Dict]: dct['argumentsIds'], dct['kwargumentsIds'], dct['serializedMemory'], + MemoryMode(dct['memoryMode']), dct['filepath'], dct['coverageId'], ) @@ -68,7 +82,7 @@ def as_execution_result(dct: Dict) -> Union[ExecutionRequest, Dict]: def parse_request(request: str) -> ExecutionRequest: - return json.loads(request, object_hook=as_execution_result) + return json.loads(request, object_hook=as_execution_request) class ResponseEncoder(json.JSONEncoder): diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py index eb0d6d5b41..bb72d56b33 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/ut_tracer.py @@ -1,6 +1,5 @@ -import dis -import inspect import logging +import math import os import pathlib import queue @@ -19,7 +18,14 @@ def _modname(path): class UtCoverageSender: - def __init__(self, coverage_id: str, host: str, port: int, use_thread: bool = False, send_coverage: bool = True): + def __init__( + self, + coverage_id: str, + host: str, + port: int, + use_thread: bool = False, + send_coverage: bool = True, + ): self.coverage_id = coverage_id self.host = host self.port = port @@ -59,10 +65,13 @@ def put_message(self, key: str): class PureSender(UtCoverageSender): def __init__(self): - super().__init__("000000", "localhost", 0, use_thread=False, send_coverage=False) + super().__init__( + "000000", "localhost", 0, use_thread=False, send_coverage=False + ) class UtTracer: + DEFAULT_LINE_FILTER = (-math.inf, math.inf) def __init__( self, tested_file: pathlib.Path, @@ -72,13 +81,16 @@ def __init__( ): self.tested_file = tested_file self.counts: dict[UtInstruction, int] = {} + self.instructions: list[UtInstruction] = [] self.localtrace = self.localtrace_count self.globaltrace = self.globaltrace_lt self.ignore_dirs = ignore_dirs self.sender = sender self.mode = mode + self.line_filter = UtTracer.DEFAULT_LINE_FILTER - def runfunc(self, func, /, *args, **kw): + def runfunc(self, func, line_filter, /, *args, **kw): + self.line_filter = line_filter result = None sys.settrace(self.globaltrace) self.f_code = func.__code__ @@ -88,39 +100,40 @@ def runfunc(self, func, /, *args, **kw): sys.settrace(None) return result - def coverage(self, filename: str) -> typing.List[int]: - filename = _modname(filename) - return [line for file, line in self.counts.keys() if file == filename] - def localtrace_count(self, frame, why, arg): filename = frame.f_code.co_filename lineno = frame.f_lineno - if pathlib.Path(filename) == self.tested_file and lineno is not None: + if ( + pathlib.Path(filename) == self.tested_file + and lineno is not None + and self.line_filter[0] <= lineno <= self.line_filter[1] + ): if self.mode == TraceMode.Instructions and frame.f_lasti is not None: offset = frame.f_lasti else: offset = 0 key = UtInstruction(lineno, offset, frame.f_code == self.f_code) - if key not in self.counts: - message = key.serialize() - try: - self.sender.put_message(message) - except Exception: - pass self.counts[key] = self.counts.get(key, 0) + 1 + self.instructions.append(key) return self.localtrace def globaltrace_lt(self, frame, why, arg): - if why == 'call': - if self.mode == TraceMode.Instructions: - frame.f_trace_opcodes = True - frame.f_trace_lines = False - filename = frame.f_globals.get('__file__', None) - if filename and all(not filename.startswith(d + os.sep) for d in self.ignore_dirs): + if why == "call": + filename = frame.f_code.co_filename + if filename and filename == str(self.tested_file.resolve()): + if self.mode == TraceMode.Instructions: + frame.f_trace_opcodes = True + frame.f_trace_lines = False + elif self.mode == TraceMode.Lines: + frame.f_trace_opcodes = False + frame.f_trace_lines = True + modulename = _modname(filename) if modulename is not None: return self.localtrace else: + frame.f_trace_opcodes = False + frame.f_trace_lines = False return None @@ -140,10 +153,11 @@ def f(x): def g(x): xs = [[j for j in range(i)] for i in range(10)] return x * 2 + return g1(x) * g(x) + 2 if __name__ == "__main__": tracer = UtTracer(pathlib.Path(__file__), [], PureSender()) - tracer.runfunc(f, 2) - print(tracer.counts) \ No newline at end of file + tracer.runfunc(f, tracer.DEFAULT_LINE_FILTER, 2) + print(tracer.counts) diff --git a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py index f5b156c0b6..d3f6d14700 100644 --- a/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py +++ b/utbot-python-executor/src/main/python/utbot_executor/utbot_executor/utils.py @@ -1,15 +1,11 @@ from __future__ import annotations import dataclasses -import dis import enum import os import sys import typing from contextlib import contextmanager -from types import CodeType, MethodType, FunctionType, LambdaType - - -InstructionType: typing.TypeAlias = MethodType | FunctionType | CodeType | type | LambdaType +from types import CodeType class TraceMode(enum.Enum): @@ -41,19 +37,8 @@ def suppress_stdout(): sys.stdout = old_stdout -def get_instructions(obj: InstructionType) -> list[UtInstruction]: - if sys.version_info >= (3, 10): - code = obj.__code__ - return [UtInstruction(line, start_offset, True) for start_offset, _, line in code.co_lines() if None not in {start_offset, line}] - else: - instructions: list[UtInstruction] = [] - current_line = None - for instruction in dis.get_instructions(obj): - if current_line is None and instruction.starts_line: - current_line = instruction.starts_line - if current_line is not None: - instructions.append(UtInstruction(instruction.starts_line, instruction.offset, True)) - return instructions +def get_instructions(obj: CodeType) -> list[UtInstruction]: + return [UtInstruction(line, start_offset, True) for start_offset, _, line in obj.co_lines() if None not in {start_offset, line}] def filter_instructions( diff --git a/utbot-python-executor/src/main/resources/utbot_executor_version b/utbot-python-executor/src/main/resources/utbot_executor_version index 9eadd6baad..60c38dfa23 100644 --- a/utbot-python-executor/src/main/resources/utbot_executor_version +++ b/utbot-python-executor/src/main/resources/utbot_executor_version @@ -1 +1 @@ -1.8.6 \ No newline at end of file +1.9.16 diff --git a/utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/PythonType.kt b/utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/PythonType.kt index e806d76f7b..0a8db02014 100644 --- a/utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/PythonType.kt +++ b/utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/PythonType.kt @@ -228,22 +228,26 @@ class PythonCallableTypeDescription( } } - fun removeNotRequiredArgs(type: UtType): FunctionType { + fun removeNotRequiredArgs(type: UtType): FunctionType? { val functionType = castToCompatibleTypeApi(type) - return createPythonCallableType( - functionType.parameters.size, - argumentKinds.filter { isRequired(it) }, - argumentNames.filterIndexed { index, _ -> isRequired(argumentKinds[index]) } - ) { self -> - val substitution = (functionType.parameters zip self.parameters).associate { - Pair(it.first as TypeParameter, it.second) + try { + return createPythonCallableType( + functionType.parameters.size, + argumentKinds.filter { isRequired(it) }, + argumentNames.filterIndexed { index, _ -> isRequired(argumentKinds[index]) } + ) { self -> + val substitution = (functionType.parameters zip self.parameters).associate { + Pair(it.first as TypeParameter, it.second) + } + FunctionTypeCreator.InitializationData( + functionType.arguments + .filterIndexed { index, _ -> isRequired(argumentKinds[index]) } + .map { DefaultSubstitutionProvider.substitute(it, substitution) }, + DefaultSubstitutionProvider.substitute(functionType.returnValue, substitution) + ) } - FunctionTypeCreator.InitializationData( - functionType.arguments - .filterIndexed { index, _ -> isRequired(argumentKinds[index]) } - .map { DefaultSubstitutionProvider.substitute(it, substitution) }, - DefaultSubstitutionProvider.substitute(functionType.returnValue, substitution) - ) + } catch (_: ClassCastException) { + return null } } } diff --git a/utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/RunMypy.kt b/utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/RunMypy.kt index 4491154bd1..bf76a12be6 100644 --- a/utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/RunMypy.kt +++ b/utbot-python-types/src/main/kotlin/org/utbot/python/newtyping/mypy/RunMypy.kt @@ -19,7 +19,7 @@ class MypyBuildDirectory( val fileForMypyStdout = File(root, mypyStdoutFilename) val fileForMypyStderr = File(root, mypyStderrFilename) val fileForMypyExitStatus = File(root, mypyExitStatusFilename) - private val dirForCache = File(root, mypyCacheDirectoryName) + val dirForCache = File(root, mypyCacheDirectoryName) init { val configContent = """ diff --git a/utbot-python-types/src/main/python/utbot_mypy_runner/pyproject.toml b/utbot-python-types/src/main/python/utbot_mypy_runner/pyproject.toml index f46aa4f534..7a805fe64b 100644 --- a/utbot-python-types/src/main/python/utbot_mypy_runner/pyproject.toml +++ b/utbot-python-types/src/main/python/utbot_mypy_runner/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "utbot_mypy_runner" -version = "0.2.15" +version = "0.2.16" description = "" authors = ["Ekaterina Tochilina "] readme = "README.md" diff --git a/utbot-python-types/src/main/python/utbot_mypy_runner/utbot_mypy_runner/extract_annotations.py b/utbot-python-types/src/main/python/utbot_mypy_runner/utbot_mypy_runner/extract_annotations.py index 5bf3b57aaa..90a0f8b5c3 100644 --- a/utbot-python-types/src/main/python/utbot_mypy_runner/utbot_mypy_runner/extract_annotations.py +++ b/utbot-python-types/src/main/python/utbot_mypy_runner/utbot_mypy_runner/extract_annotations.py @@ -1,15 +1,13 @@ import json -import typing as tp -from collections import defaultdict - import mypy.nodes import mypy.types - -import utbot_mypy_runner.mypy_main as mypy_main +import typing as tp import utbot_mypy_runner.expression_traverser as expression_traverser +import utbot_mypy_runner.mypy_main as mypy_main import utbot_mypy_runner.names -from utbot_mypy_runner.utils import get_borders +from collections import defaultdict from utbot_mypy_runner.nodes import * +from utbot_mypy_runner.utils import get_borders class ExpressionType: @@ -87,14 +85,14 @@ def get_result_from_mypy_build(build_result: mypy_main.build.BuildResult, source only_types = mypy_file.path not in source_paths try: - definition = get_definition_from_symbol_node(symbol_table_node, Meta(module), only_types) + definitions = get_definition_from_symbol_node(symbol_table_node, Meta(module), name, only_types) except NoTypeVarBindingException: # Bad definition, like this one: # https://github.com/sqlalchemy/sqlalchemy/blob/rel_2_0_20/lib/sqlalchemy/orm/mapped_collection.py#L521 - definition = None + definitions = {} - if definition is not None: - annotation_dict[module][name] = definition + for def_name, definition in definitions.items(): + annotation_dict[module][def_name] = definition def processor(line, col, end_line, end_col, type_, meta): expression_types[module_for_types].append( diff --git a/utbot-python-types/src/main/python/utbot_mypy_runner/utbot_mypy_runner/nodes.py b/utbot-python-types/src/main/python/utbot_mypy_runner/utbot_mypy_runner/nodes.py index 5530d4979f..18262195d7 100644 --- a/utbot-python-types/src/main/python/utbot_mypy_runner/utbot_mypy_runner/nodes.py +++ b/utbot-python-types/src/main/python/utbot_mypy_runner/utbot_mypy_runner/nodes.py @@ -1,11 +1,9 @@ -import typing as tp -from collections import defaultdict import copy -import sys - import mypy.nodes import mypy.types - +import sys +import typing as tp +from collections import defaultdict added_keys: tp.List[str] = [] annotation_node_dict: tp.Dict[str, "AnnotationNode"] = {} @@ -563,10 +561,24 @@ def get_definition_from_node(node: mypy.nodes.Node, meta: Meta, only_types: bool def get_definition_from_symbol_node( table_node: mypy.nodes.SymbolTableNode, meta: Meta, + base_name: str, only_types: bool = False -) -> tp.Optional[Definition]: +) -> tp.Dict[str, Definition]: if table_node.node is None or not (table_node.node.fullname.startswith(meta.module_name)) \ or not isinstance(table_node.node, mypy.nodes.Node): # this check is only for mypy - return None - - return get_definition_from_node(table_node.node, meta, only_types) + return {} + + definitions = {} + base_def = get_definition_from_node(table_node.node, meta, only_types) + if base_def is not None: + definitions[base_name] = base_def + + try: + defn = table_node.node.defn + if isinstance(defn, mypy.nodes.ClassDef): + for inner_class in filter(lambda x: isinstance(x, mypy.nodes.ClassDef), table_node.node.defn.defs.body): + definitions[f"{base_name}.{inner_class.name}"] = get_definition_from_node(inner_class.info, meta, only_types) + except AttributeError: + pass + + return definitions diff --git a/utbot-python-types/src/main/resources/utbot_mypy_runner_version b/utbot-python-types/src/main/resources/utbot_mypy_runner_version index 797eef9607..d84abc1685 100644 --- a/utbot-python-types/src/main/resources/utbot_mypy_runner_version +++ b/utbot-python-types/src/main/resources/utbot_mypy_runner_version @@ -1 +1 @@ -0.2.15 \ No newline at end of file +0.2.16 \ No newline at end of file diff --git a/utbot-python/build.gradle.kts b/utbot-python/build.gradle.kts index 8d0c4b02c6..3d0e0a3cbd 100644 --- a/utbot-python/build.gradle.kts +++ b/utbot-python/build.gradle.kts @@ -2,9 +2,6 @@ val intellijPluginVersion: String? by rootProject val kotlinLoggingVersion: String? by rootProject val apacheCommonsTextVersion: String? by rootProject val jacksonVersion: String? by rootProject -val ideType: String? by rootProject -val pythonCommunityPluginVersion: String? by rootProject -val pythonUltimatePluginVersion: String? by rootProject val log4j2Version: String? by rootProject dependencies { @@ -13,6 +10,8 @@ dependencies { api(project(":utbot-python-parser")) api(project(":utbot-python-types")) api(project(":utbot-python-executor")) + api("org.usvm:usvm-python-runner:b087c5c") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1") implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") implementation(group = "org.apache.commons", name = "commons-lang3", version = "3.12.0") @@ -27,4 +26,4 @@ dependencies { implementation(group = "org.apache.commons", name = "commons-text", version = apacheCommonsTextVersion) implementation(group = "org.apache.logging.log4j", name = "log4j-core", version = log4j2Version) implementation(group = "org.apache.logging.log4j", name = "log4j-api", version = log4j2Version) -} \ No newline at end of file +} diff --git a/utbot-python/samples/samples/algorithms/floyd_warshall.py b/utbot-python/samples/samples/algorithms/floyd_warshall.py new file mode 100644 index 0000000000..a85e042104 --- /dev/null +++ b/utbot-python/samples/samples/algorithms/floyd_warshall.py @@ -0,0 +1,42 @@ +import math + + +class Graph: + def __init__(self, n=0): # a graph with Node 0,1,...,N-1 + self.n = n + self.w = [ + [math.inf for j in range(0, n)] for i in range(0, n) + ] # adjacency matrix for weight + self.dp = [ + [math.inf for j in range(0, n)] for i in range(0, n) + ] # dp[i][j] stores minimum distance from i to j + + def add_edge(self, u, v, w): + self.dp[u][v] = w + + def floyd_warshall(self): + for k in range(0, self.n): + for i in range(0, self.n): + for j in range(0, self.n): + self.dp[i][j] = min(self.dp[i][j], self.dp[i][k] + self.dp[k][j]) + + def show_min(self, u, v): + return self.dp[u][v] + + +if __name__ == "__main__": + graph = Graph(5) + graph.add_edge(0, 2, 9) + graph.add_edge(0, 4, 10) + graph.add_edge(1, 3, 5) + graph.add_edge(2, 3, 7) + graph.add_edge(3, 0, 10) + graph.add_edge(3, 1, 2) + graph.add_edge(3, 2, 1) + graph.add_edge(3, 4, 6) + graph.add_edge(4, 1, 3) + graph.add_edge(4, 2, 4) + graph.add_edge(4, 3, 9) + graph.floyd_warshall() + graph.show_min(1, 4) + graph.show_min(0, 3) \ No newline at end of file diff --git a/utbot-python/samples/samples/easy_samples/my_func.py b/utbot-python/samples/samples/easy_samples/my_func.py new file mode 100644 index 0000000000..27b63c830b --- /dev/null +++ b/utbot-python/samples/samples/easy_samples/my_func.py @@ -0,0 +1,6 @@ +def my_func(x: int, xs: list[int]): + if len(xs) == x: + return x ** 2 + elif not xs: + return x + return len(xs) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt deleted file mode 100644 index 076194a87e..0000000000 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ /dev/null @@ -1,386 +0,0 @@ -package org.utbot.python - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.flow -import mu.KotlinLogging -import org.utbot.framework.plugin.api.* -import org.utbot.fuzzing.Control -import org.utbot.fuzzing.NoSeedValueException -import org.utbot.fuzzing.fuzz -import org.utbot.fuzzing.utils.Trie -import org.utbot.python.evaluation.EvaluationCache -import org.utbot.python.evaluation.PythonCodeExecutor -import org.utbot.python.evaluation.PythonCodeSocketExecutor -import org.utbot.python.evaluation.PythonEvaluationError -import org.utbot.python.evaluation.PythonEvaluationSuccess -import org.utbot.python.evaluation.PythonEvaluationTimeout -import org.utbot.python.evaluation.PythonWorker -import org.utbot.python.evaluation.PythonWorkerManager -import org.utbot.python.coverage.CoverageIdGenerator -import org.utbot.python.coverage.PyInstruction -import org.utbot.python.coverage.PythonCoverageMode -import org.utbot.python.coverage.buildCoverage -import org.utbot.python.evaluation.serialization.MemoryDump -import org.utbot.python.evaluation.serialization.toPythonTree -import org.utbot.python.framework.api.python.PythonTree -import org.utbot.python.framework.api.python.PythonTreeModel -import org.utbot.python.framework.api.python.PythonTreeWrapper -import org.utbot.python.framework.api.python.PythonUtExecution -import org.utbot.python.fuzzing.* -import org.utbot.python.newtyping.PythonTypeHintsStorage -import org.utbot.python.newtyping.general.UtType -import org.utbot.python.newtyping.inference.InvalidTypeFeedback -import org.utbot.python.newtyping.inference.SuccessFeedback -import org.utbot.python.newtyping.inference.baseline.BaselineAlgorithm -import org.utbot.python.newtyping.pythonModules -import org.utbot.python.newtyping.pythonTypeRepresentation -import org.utbot.python.utils.ExecutionWithTimoutMode -import org.utbot.python.utils.TestGenerationLimitManager -import org.utbot.python.utils.camelToSnakeCase -import org.utbot.summary.fuzzer.names.TestSuggestedInfo -import java.net.ServerSocket -import kotlin.random.Random - -private val logger = KotlinLogging.logger {} - -class PythonEngine( - private val methodUnderTest: PythonMethod, - private val directoriesForSysPath: Set, - private val moduleToImport: String, - private val pythonPath: String, - private val fuzzedConcreteValues: List, - private val timeoutForRun: Long, - private val pythonTypeStorage: PythonTypeHintsStorage, - private val coverageMode: PythonCoverageMode = PythonCoverageMode.Instructions, - private val sendCoverageContinuously: Boolean = true, -) { - - private val cache = EvaluationCache() - - private fun suggestExecutionName( - description: PythonMethodDescription, - executionResult: UtExecutionResult - ): TestSuggestedInfo { - val testSuffix = when (executionResult) { - is UtExecutionSuccess -> { - // can be improved - description.name - } - is UtExecutionFailure -> "${description.name}_with_exception" - else -> description.name - } - val testName = "test_$testSuffix" - return TestSuggestedInfo( - testName, - testName, - ) - } - - private fun transformModelList( - hasThisObject: Boolean, - state: MemoryDump, - modelListIds: List - ): Pair> { - val (stateThisId, resultModelListIds) = - if (hasThisObject) { - Pair(modelListIds.first(), modelListIds.drop(1)) - } else { - Pair(null, modelListIds) - } - val stateThisObject = stateThisId?.let { - PythonTreeModel( - state.getById(it).toPythonTree(state) - ) - } - val modelList = resultModelListIds.map { - PythonTreeModel( - state.getById(it).toPythonTree(state) - ) - } - return Pair(stateThisObject, modelList) - } - - private fun handleTimeoutResult( - arguments: List, - methodUnderTestDescription: PythonMethodDescription, - coveredInstructions: List, - ): FuzzingExecutionFeedback { - val summary = arguments - .zip(methodUnderTest.arguments) - .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } - val executionResult = UtTimeoutException(TimeoutException("Execution is too long")) - val testMethodName = suggestExecutionName(methodUnderTestDescription, executionResult) - - val hasThisObject = methodUnderTest.hasThisArgument - val (beforeThisObjectTree, beforeModelListTree) = if (hasThisObject) { - arguments.first() to arguments.drop(1) - } else { - null to arguments - } - val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) } - val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) } - - val coverage = Coverage(coveredInstructions) - val utFuzzedExecution = PythonUtExecution( - stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), - stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), - stateAfter = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), - diffIds = emptyList(), - result = executionResult, - coverage = coverage, - testMethodName = testMethodName.testName?.camelToSnakeCase(), - displayName = testMethodName.displayName, - summary = summary.map { DocRegularStmt(it) }, - arguments = methodUnderTest.argumentsWithoutSelf - ) - return ValidExecution(utFuzzedExecution) - } - - private fun handleSuccessResult( - arguments: List, - types: List, - evaluationResult: PythonEvaluationSuccess, - methodUnderTestDescription: PythonMethodDescription, - ): FuzzingExecutionFeedback { - val prohibitedExceptions = listOf( - "builtins.AttributeError", - "builtins.TypeError", - "builtins.NotImplementedError", - ) - - val summary = arguments - .zip(methodUnderTest.arguments) - .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } - val hasThisObject = methodUnderTest.hasThisArgument - - val resultModel = evaluationResult.stateAfter.getById(evaluationResult.resultId).toPythonTree(evaluationResult.stateAfter) - - if (evaluationResult.isException && (resultModel.type.name in prohibitedExceptions)) { // wrong type (sometimes mypy fails) - val errorMessage = "Evaluation with prohibited exception. Substituted types: ${ - types.joinToString { it.pythonTypeRepresentation() } - }. Exception type: ${resultModel.type.name}" - - logger.debug { errorMessage } - return TypeErrorFeedback(errorMessage) - } - - val executionResult = - if (evaluationResult.isException) { - UtImplicitlyThrownException(Throwable(resultModel.type.toString()), false) - } - else { - UtExecutionSuccess(PythonTreeModel(resultModel)) - } - - val testMethodName = suggestExecutionName(methodUnderTestDescription, executionResult) - - val (thisObject, initModelList) = transformModelList(hasThisObject, evaluationResult.stateInit, evaluationResult.modelListIds) - val (beforeThisObject, beforeModelList) = transformModelList(hasThisObject, evaluationResult.stateBefore, evaluationResult.modelListIds) - val (afterThisObject, afterModelList) = transformModelList(hasThisObject, evaluationResult.stateAfter, evaluationResult.modelListIds) - - val utFuzzedExecution = PythonUtExecution( - stateInit = EnvironmentModels(thisObject, initModelList, emptyMap(), executableToCall = null), - stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), - stateAfter = EnvironmentModels(afterThisObject, afterModelList, emptyMap(), executableToCall = null), - diffIds = evaluationResult.diffIds, - result = executionResult, - coverage = buildCoverage(evaluationResult.coveredStatements, evaluationResult.missedStatements), - testMethodName = testMethodName.testName?.camelToSnakeCase(), - displayName = testMethodName.displayName, - summary = summary.map { DocRegularStmt(it) }, - arguments = methodUnderTest.argumentsWithoutSelf, - ) - return ValidExecution(utFuzzedExecution) - } - - private fun constructEvaluationInput(pythonWorker: PythonWorker): PythonCodeExecutor { - return PythonCodeSocketExecutor( - methodUnderTest, - moduleToImport, - pythonPath, - directoriesForSysPath, - timeoutForRun, - pythonWorker, - ) - } - - private fun fuzzingResultHandler( - description: PythonMethodDescription, - arguments: List, - parameters: List, - manager: PythonWorkerManager, - ): PythonExecutionResult? { - val additionalModules = parameters.flatMap { it.pythonModules() } - - val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } - logger.debug(argumentValues.map { it.tree } .toString()) - val argumentModules = argumentValues - .flatMap { it.allContainingClassIds } - .map { it.moduleName } - .filterNot { it.startsWith(moduleToImport) } - val localAdditionalModules = (additionalModules + argumentModules + moduleToImport).toSet() - - val (thisObject, modelList) = if (methodUnderTest.hasThisArgument) - Pair(argumentValues[0], argumentValues.drop(1)) - else - Pair(null, argumentValues) - val functionArguments = FunctionArguments( - thisObject, - methodUnderTest.thisObjectName, - modelList, - methodUnderTest.argumentsNames - ) - try { - val coverageId = CoverageIdGenerator.createId() - return when (val evaluationResult = - manager.runWithCoverage(functionArguments, localAdditionalModules, coverageId)) { - is PythonEvaluationError -> { - val stackTraceMessage = evaluationResult.stackTrace.joinToString("\n") - val utError = UtError( - "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}\n${stackTraceMessage}", - Throwable(stackTraceMessage) - ) - description.limitManager.addInvalidExecution() - logger.debug(stackTraceMessage) - PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) - } - - is PythonEvaluationTimeout -> { - val coveredInstructions = - manager.coverageReceiver.coverageStorage.getOrDefault(coverageId, mutableListOf()) - val utTimeoutException = handleTimeoutResult(arguments, description, coveredInstructions) - val trieNode: Trie.Node = - if (coveredInstructions.isEmpty()) - Trie.emptyNode() - else - description.tracer.add(coveredInstructions) - description.limitManager.addInvalidExecution() - PythonExecutionResult( - utTimeoutException, - PythonFeedback(control = Control.PASS, result = trieNode, SuccessFeedback) - ) - } - - is PythonEvaluationSuccess -> { - val coveredInstructions = evaluationResult.coveredStatements - - val result = handleSuccessResult( - arguments, - parameters, - evaluationResult, - description, - ) - val typeInferenceFeedback = if (result is ValidExecution) SuccessFeedback else InvalidTypeFeedback - when (result) { - is ValidExecution -> { - val trieNode: Trie.Node = description.tracer.add(coveredInstructions) - description.limitManager.addSuccessExecution() - PythonExecutionResult( - result, - PythonFeedback(Control.CONTINUE, trieNode, typeInferenceFeedback) - ) - } - is InvalidExecution -> { - description.limitManager.addInvalidExecution() - PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE, typeInferenceFeedback = typeInferenceFeedback)) - } - else -> { - description.limitManager.addInvalidExecution() - PythonExecutionResult(result, PythonFeedback(control = Control.PASS, typeInferenceFeedback = typeInferenceFeedback)) - } - } - } - } - } catch (_: TimeoutException) { - logger.debug { "Fuzzing process was interrupted by timeout" } - return null - } - } - - fun fuzzing( - parameters: List, - typeInferenceAlgorithm: BaselineAlgorithm, - isCancelled: () -> Boolean, - until: Long - ): Flow = flow { - ServerSocket(0).use { serverSocket -> - logger.debug { "Server port: ${serverSocket.localPort}" } - val manager = try { - PythonWorkerManager( - serverSocket, - pythonPath, - until, - coverageMode, - sendCoverageContinuously, - ) { constructEvaluationInput(it) } - } catch (_: TimeoutException) { - return@flow - } - logger.debug { "Executor manager was created successfully" } - - val pmd = PythonMethodDescription( - methodUnderTest.name, - parameters, - fuzzedConcreteValues, - pythonTypeStorage, - Trie(PyInstruction::id), - Random(0), - TestGenerationLimitManager(ExecutionWithTimoutMode, until, isRootManager = true), - methodUnderTest.definition.type, - ) - - try { - if (parameters.isEmpty()) { - val result = fuzzingResultHandler(pmd, emptyList(), parameters, manager) - result?.let { - emit(it.fuzzingExecutionFeedback) - } - } else { - try { - PythonFuzzing(pythonTypeStorage, typeInferenceAlgorithm) { description, arguments -> - if (isCancelled()) { - logger.debug { "Fuzzing process was interrupted" } - manager.disconnect() - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } - if (System.currentTimeMillis() >= until) { - logger.debug { "Fuzzing process was interrupted by timeout" } - manager.disconnect() - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } - - if (arguments.any { PythonTree.containsFakeNode(it.tree) }) { - logger.debug { "FakeNode in Python model" } - description.limitManager.addFakeNodeExecutions() - emit(FakeNodeFeedback) - return@PythonFuzzing PythonFeedback(control = Control.CONTINUE) - } - - val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) - val mem = cache.get(pair) - if (mem != null) { - logger.debug { "Repeat in fuzzing ${arguments.map {it.tree}}" } - description.limitManager.addSuccessExecution() - emit(CachedExecutionFeedback(mem.fuzzingExecutionFeedback)) - return@PythonFuzzing mem.fuzzingPlatformFeedback.fromCache() - } - val result = fuzzingResultHandler(description, arguments, parameters, manager) - if (result == null) { // timeout - manager.disconnect() - return@PythonFuzzing PythonFeedback(control = Control.STOP) - } - - cache.add(pair, result) - emit(result.fuzzingExecutionFeedback) - return@PythonFuzzing result.fuzzingPlatformFeedback - }.fuzz(pmd) - } catch (_: NoSeedValueException) { - logger.debug { "Cannot fuzz values for types: ${parameters.map { it.pythonTypeRepresentation() }}" } - } - } - } finally { - manager.shutdown() - } - } - } -} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt index cd43648659..0c5486133f 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestCaseGenerator.kt @@ -1,340 +1,70 @@ package org.utbot.python -import kotlinx.coroutines.runBlocking import mu.KotlinLogging import org.utbot.framework.minimization.minimizeExecutions +import org.utbot.framework.plugin.api.UtClusterInfo import org.utbot.framework.plugin.api.UtError -import org.utbot.framework.plugin.api.UtExecution import org.utbot.framework.plugin.api.UtExecutionSuccess -import org.utbot.python.coverage.PythonCoverageMode +import org.utbot.framework.plugin.api.UtTimeoutException +import org.utbot.python.engine.GlobalPythonEngine import org.utbot.python.framework.api.python.PythonUtExecution -import org.utbot.python.framework.api.python.util.pythonStrClassId -import org.utbot.python.fuzzing.* -import org.utbot.python.newtyping.* -import org.utbot.python.newtyping.ast.visitor.Visitor -import org.utbot.python.newtyping.ast.visitor.constants.ConstantCollector -import org.utbot.python.newtyping.ast.visitor.hints.HintCollector -import org.utbot.python.newtyping.general.* -import org.utbot.python.newtyping.inference.baseline.BaselineAlgorithm -import org.utbot.python.newtyping.mypy.GlobalNamesStorage -import org.utbot.python.newtyping.mypy.MypyInfoBuild -import org.utbot.python.newtyping.mypy.MypyReportLine -import org.utbot.python.newtyping.mypy.getErrorNumber -import org.utbot.python.newtyping.utils.getOffsetLine -import org.utbot.python.newtyping.utils.isRequired -import org.utbot.python.utils.TestGenerationLimitManager -import org.utbot.python.utils.PriorityCartesianProduct -import org.utbot.python.utils.TimeoutMode private val logger = KotlinLogging.logger {} -private const val RANDOM_TYPE_FREQUENCY = 6 private const val MAX_EMPTY_COVERAGE_TESTS = 5 -private const val MAX_SUBSTITUTIONS = 10 class PythonTestCaseGenerator( - private val withMinimization: Boolean = true, - private val directoriesForSysPath: Set, - private val curModule: String, - private val pythonPath: String, - private val fileOfMethod: String, - private val isCancelled: () -> Boolean, - private val timeoutForRun: Long = 0, - private val sourceFileContent: String, - private val mypyStorage: MypyInfoBuild, - private val mypyReportLine: List, - private val coverageMode: PythonCoverageMode = PythonCoverageMode.Instructions, - private val sendCoverageContinuously: Boolean = true, + private val configuration: PythonTestGenerationConfig, + private val mypyConfig: MypyConfig, ) { + private val withMinimization = configuration.withMinimization - private val storageForMypyMessages: MutableList = mutableListOf() - - private fun constructCollectors( - mypyStorage: MypyInfoBuild, - typeStorage: PythonTypeHintsStorage, - method: PythonMethod - ): Pair { - - // initialize definitions first - mypyStorage.definitions[curModule]!!.values.map { def -> - def.getUtBotDefinition() - } - - val mypyExpressionTypes = mypyStorage.exprTypes[curModule]?.let { moduleTypes -> - moduleTypes.associate { - Pair(it.startOffset.toInt(), it.endOffset.toInt() + 1) to it.type.asUtBotType - } - } ?: emptyMap() - - val namesStorage = GlobalNamesStorage(mypyStorage) - val hintCollector = HintCollector(method.definition, typeStorage, mypyExpressionTypes, namesStorage, curModule) - val constantCollector = ConstantCollector(typeStorage) - val visitor = Visitor(listOf(hintCollector, constantCollector)) - visitor.visit(method.ast) - return Pair(hintCollector, constantCollector) - } - - private fun getCandidates(param: TypeParameter, typeStorage: PythonTypeHintsStorage): List { - val meta = param.pythonDescription() as PythonTypeVarDescription - return when (meta.parameterKind) { - PythonTypeVarDescription.ParameterKind.WithConcreteValues -> { - param.constraints.map { it.boundary } - } - PythonTypeVarDescription.ParameterKind.WithUpperBound -> { - typeStorage.simpleTypes.filter { - if (it.hasBoundedParameters()) - return@filter false - val bound = param.constraints.first().boundary - PythonSubtypeChecker.checkIfRightIsSubtypeOfLeft(bound, it, typeStorage) - } - } - } - } - - private fun generateTypesAfterSubstitution(type: UtType, typeStorage: PythonTypeHintsStorage): List { - val params = type.getBoundedParameters() - return PriorityCartesianProduct(params.map { getCandidates(it, typeStorage) }).getSequence().map { subst -> - DefaultSubstitutionProvider.substitute(type, (params zip subst).associate { it }) - }.take(MAX_SUBSTITUTIONS).toList() - } - - private fun substituteTypeParameters( - method: PythonMethod, - typeStorage: PythonTypeHintsStorage, - ): List { - val newClasses = method.containingPythonClass?.let { - generateTypesAfterSubstitution(it, typeStorage) - } ?: listOf(null) - return newClasses.flatMap { newClass -> - val funcType = newClass?.getPythonAttributeByName(typeStorage, method.name)?.type as? FunctionType - ?: method.definition.type - val newFuncTypes = generateTypesAfterSubstitution(funcType, typeStorage) - newFuncTypes.map { newFuncType -> - val def = PythonFunctionDefinition(method.definition.meta, newFuncType as FunctionType) - PythonMethod( - method.name, - method.moduleFilename, - newClass as? CompositeType, - method.codeAsString, - def, - method.ast - ) - } - }.take(MAX_SUBSTITUTIONS) - } - - private fun methodHandler( - method: PythonMethod, - typeStorage: PythonTypeHintsStorage, - coveredLines: MutableSet, - errors: MutableList, - executions: MutableList, - initMissingLines: Set?, - until: Long, - additionalVars: String = "", - ): Set? { // returns missing lines - val limitManager = TestGenerationLimitManager( - TimeoutMode, + fun generate(method: PythonMethod, until: Long): List { + logger.info { "Start test generation for ${method.name}" } + val engine = GlobalPythonEngine( + method = method, + configuration = configuration, + mypyConfig, until, ) - var missingLines = initMissingLines - - val (hintCollector, constantCollector) = constructCollectors(mypyStorage, typeStorage, method) - val constants = constantCollector.result - .mapNotNull { (type, value) -> - if (type.pythonTypeName() == pythonStrClassId.name && value is String) { - // Filter doctests - if (value.contains(">>>")) return@mapNotNull null - } - logger.debug { "Collected constant: ${type.pythonTypeRepresentation()}: $value" } - PythonFuzzedConcreteValue(type, value) - } - - val engine = PythonEngine( - method, - directoriesForSysPath, - curModule, - pythonPath, - constants, - timeoutForRun, - PythonTypeHintsStorage.get(mypyStorage), - coverageMode, - sendCoverageContinuously, - ) - val namesInModule = mypyStorage.names - .getOrDefault(curModule, emptyList()) - .map { it.name } - .filter { - it.length < 4 || !it.startsWith("__") || !it.endsWith("__") - } - - val algo = BaselineAlgorithm( - typeStorage, - hintCollector.result, - pythonPath, - method, - directoriesForSysPath, - curModule, - namesInModule, - getErrorNumber( - mypyReportLine, - fileOfMethod, - getOffsetLine(sourceFileContent, method.ast.beginOffset), - getOffsetLine(sourceFileContent, method.ast.endOffset) - ), - mypyStorage.buildRoot.configFile, - additionalVars, - randomTypeFrequency = RANDOM_TYPE_FREQUENCY, - dMypyTimeout = timeoutForRun, - ) - - val fuzzerCancellation = { isCancelled() || limitManager.isCancelled() } - - val initFunctionType = method.definition.type.arguments - runBlocking { - engine.fuzzing( - initFunctionType, - algo, - fuzzerCancellation, - until - ).collect { - when (it) { - is ValidExecution -> { - executions += it.utFuzzedExecution - missingLines = updateMissingLines(it.utFuzzedExecution, coveredLines, missingLines) - limitManager.addSuccessExecution() - } - is InvalidExecution -> { - errors += it.utError - limitManager.addInvalidExecution() - } - is ArgumentsTypeErrorFeedback -> { - limitManager.addInvalidExecution() - } - is TypeErrorFeedback -> { - limitManager.addInvalidExecution() - } - is CachedExecutionFeedback -> { - when (it.cachedFeedback) { - is ValidExecution -> { - limitManager.addSuccessExecution() - } - else -> { - limitManager.addInvalidExecution() - } - } - } - is FakeNodeFeedback -> { - limitManager.addFakeNodeExecutions() - } - } - limitManager.missedLines = missingLines?.size - } - } - - return missingLines - } - - fun generate(method: PythonMethod, until: Long): PythonTestSet { - storageForMypyMessages.clear() - - val typeStorage = PythonTypeHintsStorage.get(mypyStorage) - - val executions = mutableListOf() - val errors = mutableListOf() - val coveredLines = mutableSetOf() - - logger.info { "Start test generation for ${method.name}" } try { - val methodModifications = mutableSetOf>() // Set of pairs - - substituteTypeParameters(method, typeStorage).forEach { newMethod -> - createShortForm(newMethod)?.let { methodModifications.add(it) } - methodModifications.add(newMethod to "") - } - - val now = System.currentTimeMillis() - val timeout = (until - now) / methodModifications.size - var missingLines: Set? = null - methodModifications.forEach { (method, additionalVars) -> - missingLines = methodHandler( - method, - typeStorage, - coveredLines, - errors, - executions, - missingLines, - minOf(until, System.currentTimeMillis() + timeout), - additionalVars, - ) + if (configuration.checkUsvm) { + engine.debugUsvmRun() + } else { + engine.run() } } catch (_: OutOfMemoryError) { logger.debug { "Out of memory error. Stop test generation process" } } logger.info { "Collect all test executions for ${method.name}" } - val (emptyCoverageExecutions, coverageExecutions) = executions.partition { it.coverage == null } - val (successfulExecutions, failedExecutions) = coverageExecutions.partition { it.result is UtExecutionSuccess } + return listOf( + buildTestSet(method, engine.executionStorage.fuzzingExecutions, engine.executionStorage.fuzzingErrors, UtClusterInfo("FUZZER")), + buildTestSet(method, engine.executionStorage.symbolicExecutions, engine.executionStorage.symbolicErrors, UtClusterInfo("SYMBOLIC")), + ) + } - return PythonTestSet( - method, + private fun buildTestSet( + method: PythonMethod, + executions: List, + errors: List, + clusterInfo: UtClusterInfo, + ): PythonTestSet { + val notTimeout = executions.filter { it.result !is UtTimeoutException } + val (emptyCoverageExecutions, coverageExecutions) = notTimeout.partition { it.coverage == null } + val (successfulExecutions, failedExecutions) = coverageExecutions.partition { it.result is UtExecutionSuccess } + val minimized = if (withMinimization) minimizeExecutions(successfulExecutions) + minimizeExecutions(failedExecutions) + emptyCoverageExecutions.take(MAX_EMPTY_COVERAGE_TESTS) else - coverageExecutions + emptyCoverageExecutions.take(MAX_EMPTY_COVERAGE_TESTS), + coverageExecutions + emptyCoverageExecutions.take(MAX_EMPTY_COVERAGE_TESTS) + return PythonTestSet( + method, + minimized, errors, - storageForMypyMessages, executionsNumber = executions.size, + clustersInfo = listOf(Pair(clusterInfo, minimized.indices)) ) } - - /** - * Calculate a new set of missing lines in tested function - */ - private fun updateMissingLines( - execution: UtExecution, - coveredLines: MutableSet, - missingLines: Set? - ): Set { - execution.coverage?.coveredInstructions?.map { instr -> coveredLines.add(instr.lineNumber) } - val curMissing = - execution.coverage - ?.missedInstructions - ?.map { x -> x.lineNumber }?.toSet() - ?: emptySet() - return if (missingLines == null) curMissing else missingLines intersect curMissing - } - - companion object { - fun createShortForm(method: PythonMethod): Pair? { - val meta = method.definition.type.pythonDescription() as PythonCallableTypeDescription - val argKinds = meta.argumentKinds - if (argKinds.any { !isRequired(it) }) { - val originalDef = method.definition - val shortType = meta.removeNotRequiredArgs(originalDef.type) - val shortMeta = PythonFuncItemDescription( - originalDef.meta.name, - originalDef.meta.args.filterIndexed { index, _ -> isRequired(argKinds[index]) } - ) - val additionalVars = originalDef.meta.args - .filterIndexed { index, _ -> !isRequired(argKinds[index]) } - .mapIndexed { index, arg -> - "${arg.name}: ${method.argumentsWithoutSelf[index].annotation ?: pythonAnyType.pythonTypeRepresentation()}" - } - .joinToString(separator = "\n", prefix = "\n") - val shortDef = PythonFunctionDefinition(shortMeta, shortType) - val shortMethod = PythonMethod( - method.name, - method.moduleFilename, - method.containingPythonClass, - method.codeAsString, - shortDef, - method.ast - ) - return Pair(shortMethod, additionalVars) - } - return null - } - } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt index 99a9500e18..41d493021e 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt @@ -4,6 +4,9 @@ import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour import org.utbot.framework.codegen.domain.TestFramework import org.utbot.python.coverage.CoverageOutputFormat import org.utbot.python.coverage.PythonCoverageMode +import org.utbot.python.newtyping.mypy.MypyBuildDirectory +import org.utbot.python.newtyping.mypy.MypyInfoBuild +import org.utbot.python.newtyping.mypy.MypyReportLine import java.nio.file.Path data class TestFileInformation( @@ -20,11 +23,41 @@ class PythonTestGenerationConfig( val timeout: Long, val timeoutForRun: Long, val testFramework: TestFramework, - val testSourceRootPath: Path, + val testSourceRootPath: Path?, val withMinimization: Boolean, val isCanceled: () -> Boolean, val runtimeExceptionTestsBehaviour: RuntimeExceptionTestsBehaviour, val coverageMeasureMode: PythonCoverageMode = PythonCoverageMode.Instructions, val sendCoverageContinuously: Boolean = true, val coverageOutputFormat: CoverageOutputFormat = CoverageOutputFormat.Lines, -) \ No newline at end of file + val usvmConfig: UsvmConfig = UsvmConfig.defaultConfig, + val prohibitedExceptions: List = defaultProhibitedExceptions, + val checkUsvm: Boolean = false, + val doNotGenerateStateAssertions: Boolean = false, +) { + companion object { + val defaultProhibitedExceptions = listOf( + "builtins.AttributeError", + "builtins.TypeError", + "builtins.NotImplementedError", + ) + } +} + +data class MypyConfig( + val mypyStorage: MypyInfoBuild, + val mypyReportLine: List, + val mypyBuildDirectory: MypyBuildDirectory, +) + +data class UsvmConfig( + val javaCmd: String, + val usvmDirectory: String +) { + companion object { + val usvmDirectory = + "utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/usvm-python" + val usvmJavaCmd = "${System.getProperty("java.home")}/bin/java" + val defaultConfig = UsvmConfig(usvmJavaCmd, usvmDirectory) + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt index cd8c835e45..24decdc574 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationProcessor.kt @@ -5,7 +5,6 @@ import org.parsers.python.PythonParser import org.utbot.framework.codegen.domain.HangingTestsTimeout import org.utbot.framework.codegen.domain.models.CgMethodTestSet import org.utbot.framework.plugin.api.ExecutableId -import org.utbot.framework.plugin.api.UtClusterInfo import org.utbot.framework.plugin.api.UtExecutionSuccess import org.utbot.framework.plugin.api.util.UtContext import org.utbot.framework.plugin.api.util.withUtContext @@ -30,15 +29,18 @@ import org.utbot.python.framework.codegen.model.PythonImport import org.utbot.python.framework.codegen.model.PythonSysPathImport import org.utbot.python.framework.codegen.model.PythonSystemImport import org.utbot.python.framework.codegen.model.PythonUserImport +import org.utbot.python.newtyping.PythonConcreteCompositeTypeDescription import org.utbot.python.newtyping.PythonFunctionDefinition import org.utbot.python.newtyping.general.CompositeType import org.utbot.python.newtyping.getPythonAttributes import org.utbot.python.newtyping.mypy.MypyBuildDirectory import org.utbot.python.newtyping.mypy.MypyInfoBuild -import org.utbot.python.newtyping.mypy.MypyReportLine import org.utbot.python.newtyping.mypy.readMypyAnnotationStorageAndInitialErrors +import org.utbot.python.newtyping.pythonDescription import org.utbot.python.newtyping.pythonName import org.utbot.python.utils.TemporaryFileManager +import org.utbot.python.utils.convertToTime +import org.utbot.python.utils.separateTimeout import java.nio.file.Path import kotlin.io.path.Path import kotlin.io.path.pathString @@ -48,42 +50,41 @@ private val logger = KotlinLogging.logger {} // TODO: add asserts that one or less of containing classes and only one file abstract class PythonTestGenerationProcessor { abstract val configuration: PythonTestGenerationConfig - private val mypyBuildRoot = TemporaryFileManager.assignTemporaryFile(tag = "mypyBuildRoot") - fun sourceCodeAnalyze(): Pair> { - return readMypyAnnotationStorageAndInitialErrors( + fun sourceCodeAnalyze(): MypyConfig { + return sourceCodeAnalyze( + configuration.sysPathDirectories, configuration.pythonPath, - configuration.testFileInformation.testedFilePath, - configuration.testFileInformation.moduleName, - MypyBuildDirectory(mypyBuildRoot, configuration.sysPathDirectories) + configuration.testFileInformation, ) } - fun testGenerate(mypyStorage: MypyInfoBuild): List { - val startTime = System.currentTimeMillis() - + fun testGenerate(mypyConfig: MypyConfig): List { val testCaseGenerator = PythonTestCaseGenerator( - withMinimization = configuration.withMinimization, - directoriesForSysPath = configuration.sysPathDirectories, - curModule = configuration.testFileInformation.moduleName, - pythonPath = configuration.pythonPath, - fileOfMethod = configuration.testFileInformation.testedFilePath, - isCancelled = configuration.isCanceled, - timeoutForRun = configuration.timeoutForRun, - sourceFileContent = configuration.testFileInformation.testedFileContent, - mypyStorage = mypyStorage, - mypyReportLine = emptyList(), - coverageMode = configuration.coverageMeasureMode, - sendCoverageContinuously = configuration.sendCoverageContinuously, + configuration = configuration, + mypyConfig = mypyConfig, ) - val until = startTime + configuration.timeout + val oneFunctionTimeout = separateTimeout(configuration.timeout, configuration.testedMethods.size) + val countOfFunctions = configuration.testedMethods.size + val startTime = System.currentTimeMillis() + val tests = configuration.testedMethods.mapIndexedNotNull { index, methodHeader -> - val methodsLeft = configuration.testedMethods.size - index - val localUntil = (until - System.currentTimeMillis()) / methodsLeft + System.currentTimeMillis() + if (configuration.isCanceled()) { + return emptyList() + } + val usedTime = System.currentTimeMillis() - startTime + val expectedTime = index * oneFunctionTimeout + val localOneFunctionTimeout = if (usedTime < expectedTime) { + separateTimeout(configuration.timeout - usedTime, countOfFunctions - index) + } else { + oneFunctionTimeout + } + val localUntil = System.currentTimeMillis() + localOneFunctionTimeout + logger.info { "Local timeout ${configuration.timeout / configuration.testedMethods.size}ms. Until ${localUntil.convertToTime()}" } try { val method = findMethodByHeader( - mypyStorage, + mypyConfig.mypyStorage, methodHeader, configuration.testFileInformation.moduleName, configuration.testFileInformation.testedFileContent @@ -93,24 +94,35 @@ abstract class PythonTestGenerationProcessor { logger.warn { "Skipping method ${e.methodName}: did not find its function definition" } null } - } - val (notEmptyTests, emptyTestSets) = tests.partition { it.executions.isNotEmpty() } + }.flatten() + val notEmptyTests = tests.filter { it.executions.isNotEmpty() } + + val emptyTests = tests + .groupBy { it.method } + .filter { it.value.all { testSet -> testSet.executions.isEmpty() } } + .map { it.key.name } - if (emptyTestSets.isNotEmpty()) { - notGeneratedTestsAction(emptyTestSets.map { it.method.name }) + if (emptyTests.isNotEmpty()) { + notGeneratedTestsAction(emptyTests) } return notEmptyTests } - fun testCodeGenerate(testSets: List): String { + fun testCodeGenerateSplitImports(testSets: List): Pair> { + val allImports = collectImports(testSets) + val code = testCodeGenerate(testSets, true) + return code to allImports + } + + fun testCodeGenerate(testSets: List, skipImports: Boolean = false): String { val containingClassName = getContainingClassName(testSets) val classId = PythonClassId(configuration.testFileInformation.moduleName, containingClassName) val methodIds = testSets.associate { testSet -> testSet.method to PythonMethodId( classId, - testSet.method.name, + testSet.method.renderMethodName(), RawPythonAnnotation(pythonAnyClassId.name), testSet.method.arguments.map { argument -> argument.annotation?.let { annotation -> @@ -128,7 +140,8 @@ abstract class PythonTestGenerationProcessor { methodIds[testSet.method] as ExecutableId to params }.toMutableMap() - val allImports = collectImports(testSets) + val collectedImports = collectImports(testSets) + val allImports = if (skipImports) emptySet() else collectedImports val context = UtContext(this::class.java.classLoader) withUtContext(context) { @@ -140,14 +153,13 @@ abstract class PythonTestGenerationProcessor { hangingTestsTimeout = HangingTestsTimeout(configuration.timeoutForRun), runtimeExceptionTestsBehaviour = configuration.runtimeExceptionTestsBehaviour, ) + codegen.context.existingVariableNames = codegen.context.existingVariableNames.addAll(collectedImports.flatMap { listOfNotNull(it.moduleName, it.rootModuleName, it.importName) }) val testCode = codegen.pythonGenerateAsStringWithTestReport( testSets.map { testSet -> - val intRange = testSet.executions.indices - val clusterInfo = listOf(Pair(UtClusterInfo("FUZZER"), intRange)) CgMethodTestSet( executableId = methodIds[testSet.method] as ExecutableId, executions = testSet.executions, - clustersInfo = clusterInfo, + clustersInfo = testSet.clustersInfo, ) }, allImports @@ -163,7 +175,7 @@ abstract class PythonTestGenerationProcessor { abstract fun processCoverageInfo(testSets: List) private fun getContainingClassName(testSets: List): String { - val containingClasses = testSets.map { it.method.containingPythonClass?.pythonName() ?: "TopLevelFunctions" } + val containingClasses = testSets.map { it.method.containingPythonClass?.pythonName()?.replace(".", "") ?: "TopLevelFunctions" } return containingClasses.toSet().first() } @@ -228,27 +240,46 @@ abstract class PythonTestGenerationProcessor { ): PythonMethod { var containingClass: CompositeType? = null val containingClassName = method.containingPythonClassId?.simpleName - val functionDef = if (containingClassName == null) { - mypyStorage.definitions[curModule]!![method.name]!!.getUtBotDefinition()!! + val definition = if (containingClassName == null) { + mypyStorage.definitions[curModule]?.get(method.name)?.getUtBotDefinition() } else { containingClass = - mypyStorage.definitions[curModule]!![containingClassName]!!.getUtBotType() as CompositeType - mypyStorage.definitions[curModule]!![containingClassName]!!.type.asUtBotType.getPythonAttributes().first { + mypyStorage.definitions[curModule]?.get(containingClassName)?.getUtBotType() as? CompositeType + ?: throw SelectedMethodIsNotAFunctionDefinition(method.name) + val descr = containingClass.pythonDescription() + if (descr !is PythonConcreteCompositeTypeDescription) + throw SelectedMethodIsNotAFunctionDefinition(method.name) + mypyStorage.definitions[curModule]?.get(containingClassName)?.type?.asUtBotType?.getPythonAttributes()?.first { it.meta.name == method.name } - } as? PythonFunctionDefinition ?: throw SelectedMethodIsNotAFunctionDefinition(method.name) - + } ?: throw SelectedMethodIsNotAFunctionDefinition(method.name) val parsedFile = PythonParser(sourceFileContent).Module() val funcDef = PythonCode.findFunctionDefinition(parsedFile, method) + val decorators = funcDef.decorators.map { PyDecorator.decoratorByName(it.name.toString()) } + + if (definition is PythonFunctionDefinition) { + return PythonBaseMethod( + name = method.name, + moduleFilename = method.moduleFilename, + containingPythonClass = containingClass, + codeAsString = funcDef.body.source, + definition = definition, + ast = funcDef.body + ) + } else if (decorators == listOf(PyDecorator.StaticMethod)) { + return PythonDecoratedMethod( + name = method.name, + moduleFilename = method.moduleFilename, + containingPythonClass = containingClass, + codeAsString = funcDef.body.source, + definition = definition, + ast = funcDef.body, + decorator = decorators.first() + ) + } else { + throw SelectedMethodIsNotAFunctionDefinition(method.name) + } - return PythonMethod( - name = method.name, - moduleFilename = method.moduleFilename, - containingPythonClass = containingClass, - codeAsString = funcDef.body.source, - definition = functionDef, - ast = funcDef.body - ) } private fun relativizePaths(rootPath: Path?, paths: Set): Set = @@ -292,6 +323,25 @@ abstract class PythonTestGenerationProcessor { val notCovered = coverageInfo.notCovered.map { it.toJson() } return "{\"covered\": [${covered.joinToString(", ")}], \"notCovered\": [${notCovered.joinToString(", ")}]}" } + + companion object { + fun sourceCodeAnalyze( + sysPathDirectories: Set, + pythonPath: String, + testFileInformation: TestFileInformation, + ): MypyConfig { + val mypyBuildRoot = TemporaryFileManager.assignTemporaryFile(tag = "mypyBuildRoot") + val buildDirectory = MypyBuildDirectory(mypyBuildRoot, sysPathDirectories) + val (mypyInfoBuild, mypyReportLines) = readMypyAnnotationStorageAndInitialErrors( + pythonPath, + testFileInformation.testedFilePath, + testFileInformation.moduleName, + buildDirectory + ) + return MypyConfig(mypyInfoBuild, mypyReportLines, buildDirectory) + } + + } } -data class SelectedMethodIsNotAFunctionDefinition(val methodName: String): Exception() \ No newline at end of file +data class SelectedMethodIsNotAFunctionDefinition(val methodName: String): Exception() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt b/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt index 57da4cc17d..c439f7e00d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/UTPythonAPI.kt @@ -1,6 +1,7 @@ package org.utbot.python import org.parsers.python.ast.Block +import org.utbot.framework.plugin.api.UtClusterInfo import org.utbot.framework.plugin.api.UtError import org.utbot.framework.plugin.api.UtExecution import org.utbot.python.framework.api.python.PythonClassId @@ -8,8 +9,9 @@ import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.newtyping.* import org.utbot.python.newtyping.general.CompositeType -import org.utbot.python.newtyping.mypy.MypyReportLine +import org.utbot.python.newtyping.general.FunctionType import org.utbot.python.newtyping.utils.isNamed +import org.utbot.python.newtyping.utils.isRequired data class PythonArgument( val name: String, @@ -20,19 +22,45 @@ data class PythonArgument( class PythonMethodHeader( val name: String, val moduleFilename: String, - val containingPythonClassId: PythonClassId? + val containingPythonClassId: PythonClassId?, + val decorators: List = emptyList(), ) -class PythonMethod( - val name: String, - val moduleFilename: String, - val containingPythonClass: CompositeType?, - val codeAsString: String, - var definition: PythonFunctionDefinition, + +interface PythonMethod { + val name: String + val moduleFilename: String + val containingPythonClass: CompositeType? + val codeAsString: String val ast: Block -) { - fun methodSignature(): String = "$name(" + arguments.joinToString(", ") { + fun methodSignature(): String + val hasThisArgument: Boolean + val arguments: List + val argumentsWithoutSelf: List + val thisObjectName: String? + val argumentsNames: List + val argumentsNamesWithoutSelf: List + + val methodType: FunctionType + val methodMeta: PythonDefinitionDescription + + fun makeCopyWithNewType(newFunctionType: FunctionType): PythonMethod + fun createShortForm(): Pair? + fun changeDefinition(signature: FunctionType) + + fun renderMethodName(): String +} + +class PythonBaseMethod( + override val name: String, + override val moduleFilename: String, + override val containingPythonClass: CompositeType?, + override val codeAsString: String, + private var definition: PythonFunctionDefinition, + override val ast: Block +) : PythonMethod { + override fun methodSignature(): String = "$name(" + arguments.joinToString(", ") { "${it.name}: ${it.annotation ?: pythonAnyClassId.name}" } + ")" @@ -40,38 +68,182 @@ class PythonMethod( Check that the first argument is `self` of `cls`. TODO: We should support `@property` decorator */ - val hasThisArgument: Boolean + override val hasThisArgument: Boolean get() = containingPythonClass != null && definition.meta.args.any { it.isSelf } - val arguments: List + override val arguments: List + get() { + val meta = definition.meta + val description = definition.type.pythonDescription() as PythonCallableTypeDescription + return (definition.type.arguments).mapIndexed { index, type -> + PythonArgument( + meta.args[index].name, + type.pythonTypeRepresentation(), // TODO: improve pythonTypeRepresentation + isNamed(description.argumentKinds[index]) + ) + } + } + + override val argumentsWithoutSelf: List + get() = if (hasThisArgument) arguments.drop(1) else arguments + + override val thisObjectName: String? + get() = if (hasThisArgument) arguments[0].name else null + + override val argumentsNames: List + get() = arguments.map { it.name } + + override val argumentsNamesWithoutSelf: List + get() = argumentsNames.drop(if (hasThisArgument) 1 else 0) + + override val methodType: FunctionType = definition.type + + override val methodMeta: PythonDefinitionDescription = definition.meta + + override fun makeCopyWithNewType(newFunctionType: FunctionType): PythonMethod { + val newDefinition = PythonFunctionDefinition(definition.meta, newFunctionType) + return PythonBaseMethod(name, moduleFilename, containingPythonClass, codeAsString, newDefinition, ast) + } + + override fun createShortForm(): Pair? { + val meta = methodType.pythonDescription() as PythonCallableTypeDescription + val argKinds = meta.argumentKinds + if (argKinds.any { !isRequired(it) }) { + val originalDef = definition + val shortType = meta.removeNotRequiredArgs(originalDef.type) ?: return null + val shortMeta = PythonFuncItemDescription( + originalDef.meta.name, + originalDef.meta.args.filterIndexed { index, _ -> isRequired(argKinds[index]) } + ) + val additionalVars = originalDef.meta.args + .filterIndexed { index, _ -> !isRequired(argKinds[index]) } + .mapIndexed { index, arg -> + "${arg.name}: ${argumentsWithoutSelf[index].annotation ?: pythonAnyType.pythonTypeRepresentation()}" + } + .joinToString(separator = "\n", prefix = "\n") + val shortDef = PythonFunctionDefinition(shortMeta, shortType) + val shortMethod = PythonBaseMethod( + name, + moduleFilename, + containingPythonClass, + codeAsString, + shortDef, + ast + ) + return Pair(shortMethod, additionalVars) + } + return null + } + + fun changeDefinition(newDefinition: PythonDefinition) { + require(newDefinition is PythonFunctionDefinition) + definition = newDefinition + } + + override fun changeDefinition(signature: FunctionType) { + val newDefinition = PythonFunctionDefinition( + definition.meta, + signature + ) + changeDefinition(newDefinition) + } + + override fun renderMethodName(): String { + return name + } +} + +class PythonDecoratedMethod( + override val name: String, + override val moduleFilename: String, + override val containingPythonClass: CompositeType?, + override val codeAsString: String, + private var definition: PythonDefinition, + override val ast: Block, + val decorator: PyDecorator, +) : PythonMethod { + override val methodType: FunctionType = definition.type as FunctionType + override val methodMeta: PythonDefinitionDescription = definition.meta + val typeMeta: PythonCallableTypeDescription = definition.type.pythonDescription() as PythonCallableTypeDescription + + fun changeDefinition(newDefinition: PythonDefinition) { + require(checkDefinition(newDefinition)) { error("Cannot test non-function object") } + definition = newDefinition + } + + override fun changeDefinition(signature: FunctionType) { + val newDefinition = PythonDefinition( + methodMeta, + signature + ) + checkDefinition(newDefinition) + changeDefinition(newDefinition) + } + + override fun renderMethodName(): String { + return decorator.generateCallableName(this) + } + + override fun makeCopyWithNewType(newFunctionType: FunctionType): PythonMethod { + val newDefinition = PythonDefinition(methodMeta, newFunctionType) + return PythonDecoratedMethod( + name, moduleFilename, containingPythonClass, codeAsString, newDefinition, ast, decorator + ) + } + + override fun createShortForm(): Pair? = null + + init { + assert(checkDefinition(definition)) { error("Cannot test non-function object") } + } + override fun methodSignature(): String = "${decorator.generateCallableName(this)}(" + arguments.joinToString(", ") { + "${it.name}: ${it.annotation ?: pythonAnyClassId.name}" + } + ")" + + /* + Check that the first argument is `self` of `cls`. + */ + override val hasThisArgument: Boolean + get() = containingPythonClass != null && decorator.hasSelfArgument() + + override val arguments: List get() { - val meta = definition.type.pythonDescription() as PythonCallableTypeDescription - return (definition.type.arguments).mapIndexed { index, type -> + return (methodType.arguments).mapIndexed { index, type -> PythonArgument( - meta.argumentNames[index]!!, + typeMeta.argumentNames[index] ?: "arg$index", type.pythonTypeRepresentation(), // TODO: improve pythonTypeRepresentation - isNamed(meta.argumentKinds[index]) + isNamed(typeMeta.argumentKinds[index]) ) } } - val argumentsWithoutSelf: List + override val argumentsWithoutSelf: List get() = if (hasThisArgument) arguments.drop(1) else arguments - val thisObjectName: String? + override val thisObjectName: String? get() = if (hasThisArgument) arguments[0].name else null - val argumentsNames: List - get() = arguments.map { it.name }.drop(if (hasThisArgument) 1 else 0) + override val argumentsNames: List + get() = arguments.map { it.name } + + override val argumentsNamesWithoutSelf: List + get() = argumentsNames.drop(if (hasThisArgument) 1 else 0) + + companion object { + fun checkDefinition(definition: PythonDefinition): Boolean { + val type = definition.type + val meta = definition.type.pythonDescription() + return type is FunctionType && meta is PythonCallableTypeDescription + } + } } data class PythonTestSet( val method: PythonMethod, val executions: List, val errors: List, - val mypyReport: List, - val classId: PythonClassId? = null, val executionsNumber: Int = 0, + val clustersInfo: List> = listOf(null to executions.indices) ) data class FunctionArguments( @@ -81,4 +253,47 @@ data class FunctionArguments( val names: List, ) { val allArguments: List = (listOf(thisObject) + arguments).filterNotNull() +} + +sealed interface PyDecorator { + fun generateCallableName(method: PythonMethod, baseName: String? = null): String + fun hasSelfArgument(): Boolean + val type: PythonClassId + + object StaticMethod : PyDecorator { + override fun generateCallableName(method: PythonMethod, baseName: String?) = + "${method.containingPythonClass!!.pythonName()}.${method.name}" + + override fun hasSelfArgument() = false + + override val type: PythonClassId = PythonClassId("staticmethod") + } + + object ClassMethod : PyDecorator { + override fun generateCallableName(method: PythonMethod, baseName: String?) = + "${method.containingPythonClass!!.pythonName()}.${method.name}" + + override fun hasSelfArgument() = true + + override val type: PythonClassId = PythonClassId("classmethod") + } + + class UnknownDecorator( + override val type: PythonClassId, + ) : PyDecorator { + override fun generateCallableName(method: PythonMethod, baseName: String?) = + baseName ?: method.name + + override fun hasSelfArgument() = true + } + + companion object { + fun decoratorByName(decoratorName: String): PyDecorator { + return when (decoratorName) { + "classmethod" -> ClassMethod + "staticmethod" -> StaticMethod + else -> UnknownDecorator(PythonClassId(decoratorName)) + } + } + } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/code/PythonCodeAPI.kt b/utbot-python/src/main/kotlin/org/utbot/python/code/PythonCodeAPI.kt index aa8d2ed873..10cd2c6031 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/code/PythonCodeAPI.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/code/PythonCodeAPI.kt @@ -2,8 +2,8 @@ package org.utbot.python.code import org.parsers.python.ast.Block import org.parsers.python.ast.ClassDefinition -import org.parsers.python.ast.Module import org.parsers.python.ast.FunctionDefinition +import org.parsers.python.ast.Module import org.utbot.python.PythonMethodHeader import org.utbot.python.newtyping.ast.ParsedFunctionDefinition import org.utbot.python.newtyping.ast.parseClassDefinition @@ -19,8 +19,12 @@ object PythonCode { return parsedFile.children().filterIsInstance() } - fun getClassMethods(class_: Block): List { - return class_.children().filterIsInstance() + fun getInnerClasses(classDef: ClassDefinition): List { + return classDef.children().filterIsInstance().flatMap {it.children() }.filterIsInstance() + } + + fun getClassMethods(classBlock: Block): List { + return classBlock.children().filterIsInstance() } fun findFunctionDefinition(parsedFile: Module, method: PythonMethodHeader): ParsedFunctionDefinition { @@ -32,6 +36,7 @@ object PythonCode { } ?: throw Exception("Couldn't find top-level function ${method.name}") } else { getTopLevelClasses(parsedFile) + .flatMap { listOf(it) + getInnerClasses(it) } .mapNotNull { parseClassDefinition(it) } .flatMap { getClassMethods(it.body) } .mapNotNull { parseFunctionDefinition(it) } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt index 34403ad42a..a9fe85b3d6 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/coverage/CoverageApi.kt @@ -36,6 +36,18 @@ data class PyInstruction( constructor(lineNumber: Int, id: Long) : this(lineNumber, id.floorDiv(2).toPair().second, id % 2 == 1L) } +data class PyInstructionEdge( + val instruction1: PyInstruction, + val instruction2: PyInstruction, +) : Instruction( + "", + "", + instruction2.pyLineNumber, + (instruction1.id to instruction2.id).toCoverageId() +) { + override fun toString(): String = "$instruction1 -> $instruction2" +} + fun Boolean.toLong() = if (this) 1L else 0L fun String.toPyInstruction(): PyInstruction? { @@ -61,8 +73,17 @@ fun String.toPyInstruction(): PyInstruction? { } fun buildCoverage(coveredStatements: List, missedStatements: List): Coverage { + return buildEdgeCoverage(coveredStatements, missedStatements) +// return Coverage( +// coveredInstructions = coveredStatements, +// instructionsCount = (coveredStatements.size + missedStatements.size).toLong(), +// missedInstructions = missedStatements +// ) +} + +fun buildEdgeCoverage(coveredStatements: List, missedStatements: List): Coverage { return Coverage( - coveredInstructions = coveredStatements, + coveredInstructions = coveredStatements.windowed(2, 1).map { PyInstructionEdge(it[0], it[1]) }, instructionsCount = (coveredStatements.size + missedStatements.size).toLong(), missedInstructions = missedStatements ) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/engine/ExecutionFeedback.kt b/utbot-python/src/main/kotlin/org/utbot/python/engine/ExecutionFeedback.kt new file mode 100644 index 0000000000..79e43604c8 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/engine/ExecutionFeedback.kt @@ -0,0 +1,12 @@ +package org.utbot.python.engine + +import org.utbot.framework.plugin.api.UtError +import org.utbot.python.framework.api.python.PythonUtExecution + +sealed interface ExecutionFeedback +class ValidExecution(val utFuzzedExecution: PythonUtExecution): ExecutionFeedback +class InvalidExecution(val utError: UtError): ExecutionFeedback +class TypeErrorFeedback(val message: String) : ExecutionFeedback +class ArgumentsTypeErrorFeedback(val message: String) : ExecutionFeedback +class CachedExecutionFeedback(val cachedFeedback: ExecutionFeedback) : ExecutionFeedback +object FakeNodeFeedback : ExecutionFeedback \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/engine/ExecutionStorage.kt b/utbot-python/src/main/kotlin/org/utbot/python/engine/ExecutionStorage.kt new file mode 100644 index 0000000000..c5ff098d6b --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/engine/ExecutionStorage.kt @@ -0,0 +1,36 @@ +package org.utbot.python.engine + +import org.utbot.framework.plugin.api.UtError +import org.utbot.python.framework.api.python.PythonUtExecution + +class ExecutionStorage { + val fuzzingExecutions: MutableList = mutableListOf() + val fuzzingErrors: MutableList = mutableListOf() + + val symbolicExecutions: MutableList = mutableListOf() + val symbolicErrors: MutableList = mutableListOf() + + fun saveFuzzingExecution(feedback: ExecutionFeedback) { + when (feedback) { + is ValidExecution -> { + fuzzingExecutions += feedback.utFuzzedExecution + } + is InvalidExecution -> { + fuzzingErrors += feedback.utError + } + else -> {} + } + } + + fun saveSymbolicExecution(feedback: ExecutionFeedback) { + when (feedback) { + is ValidExecution -> { + symbolicExecutions += feedback.utFuzzedExecution + } + is InvalidExecution -> { + symbolicErrors += feedback.utError + } + else -> {} + } + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt new file mode 100644 index 0000000000..ae1982ac8c --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt @@ -0,0 +1,167 @@ +package org.utbot.python.engine + +import mu.KotlinLogging +import org.usvm.runner.StandardLayout +import org.usvm.runner.USVMPythonConfig +import org.usvm.runner.USVMPythonFunctionConfig +import org.usvm.runner.USVMPythonMethodConfig +import org.usvm.runner.USVMPythonRunConfig +import org.usvm.runner.venv.extractVenvConfig +import org.utbot.python.MypyConfig +import org.utbot.python.PythonMethod +import org.utbot.python.PythonTestGenerationConfig +import org.utbot.python.engine.fuzzing.FuzzingEngine +import org.utbot.python.engine.symbolic.SymbolicEngine +import org.utbot.python.engine.symbolic.SymbolicExecutionEvaluator +import org.utbot.python.engine.symbolic.USVMPythonAnalysisResultReceiverImpl +import org.utbot.python.newtyping.PythonTypeHintsStorage +import org.utbot.python.newtyping.ast.visitor.Visitor +import org.utbot.python.newtyping.ast.visitor.constants.ConstantCollector +import org.utbot.python.newtyping.ast.visitor.hints.HintCollector +import org.utbot.python.newtyping.mypy.GlobalNamesStorage +import org.utbot.python.newtyping.mypy.MypyInfoBuild +import org.utbot.python.newtyping.pythonName +import org.utbot.python.utils.convertToTime +import java.io.File +import kotlin.concurrent.thread + +private val logger = KotlinLogging.logger {} + +class GlobalPythonEngine( + val method: PythonMethod, + val configuration: PythonTestGenerationConfig, + private val mypyConfig: MypyConfig, + val until: Long, +) { + val executionStorage = ExecutionStorage() + private val typeStorage = PythonTypeHintsStorage.get(mypyConfig.mypyStorage) + private val constantCollector = ConstantCollector(typeStorage) + private val hintCollector = constructHintCollector( + mypyConfig.mypyStorage, + typeStorage, + constantCollector, + method, + configuration.testFileInformation.moduleName + ) + + private fun runFuzzing() { + FuzzingEngine( + method, + configuration, + typeStorage, + hintCollector, + constantCollector, + mypyConfig.mypyStorage, + mypyConfig.mypyReportLine, + until, + executionStorage, + ).start() + } + + private fun runSymbolic(debug: Boolean = false) { + logger.info { "Symbolic until: ${until.convertToTime()}" } + val usvmPythonConfig = USVMPythonConfig( + StandardLayout(File(configuration.usvmConfig.usvmDirectory)), + configuration.usvmConfig.javaCmd, + mypyConfig.mypyBuildDirectory.root.canonicalPath, + configuration.sysPathDirectories, + extractVenvConfig(configuration.pythonPath) + ) + val runner = SymbolicExecutionEvaluator( + method, + configuration, + executionStorage, + until + ) + runner.start() + val receiver = USVMPythonAnalysisResultReceiverImpl(runner.executionQueue) + val config = if (method.containingPythonClass == null) { + USVMPythonFunctionConfig(configuration.testFileInformation.moduleName, method.name) + } else { + USVMPythonMethodConfig( + configuration.testFileInformation.moduleName, + method.name, + method.containingPythonClass!!.pythonName() + ) + } + val engine = SymbolicEngine( + usvmPythonConfig, + configuration, + ) + val usvmConfig = USVMPythonRunConfig(config, until - System.currentTimeMillis(), configuration.timeoutForRun * 2) + if (debug) { + engine.debugRun(usvmConfig) + } else { + engine.analyze( + usvmConfig, + receiver + ) + } + logger.info { "Symbolic: stopped receiver" } + runner.receiverFinished = true + runner.join() + runner.close() + logger.info { "Symbolic: stopped runner" } + } + + fun run() { + val fuzzing = thread( + start = true, + isDaemon = false, + name = "Fuzzer" + ) { + logger.info { " >>>>>>> Start fuzzer >>>>>>> " } + runFuzzing() + logger.info { " <<<<<<< Finish fuzzer <<<<<<< " } + } + val symbolic = thread( + start = true, + isDaemon = false, + name = "Symbolic" + ) { + logger.info { " ------- Start symbolic ------- " } + runSymbolic() + logger.info { " ======= Finish symbolic ======= " } + } + fuzzing.join() + symbolic.join() + } + + fun debugUsvmRun() { + val symbolic = thread( + start = true, + isDaemon = true, + name = "Symbolic" + ) { + logger.info { " ...... Checking symbolic ...... " } + runSymbolic(debug = true) + } + symbolic.join() + } + + private fun constructHintCollector( + mypyStorage: MypyInfoBuild, + typeStorage: PythonTypeHintsStorage, + constantCollector: ConstantCollector, + method: PythonMethod, + moduleName: String, + ): HintCollector { + + // initialize definitions first + mypyStorage.definitions[moduleName]!!.values.map { def -> + def.getUtBotDefinition() + } + + val mypyExpressionTypes = mypyStorage.exprTypes[moduleName]?.let { moduleTypes -> + moduleTypes.associate { + Pair(it.startOffset.toInt(), it.endOffset.toInt() + 1) to it.type.asUtBotType + } + } ?: emptyMap() + + val namesStorage = GlobalNamesStorage(mypyStorage) + val hintCollector = HintCollector(method, typeStorage, mypyExpressionTypes, namesStorage, moduleName) + val visitor = Visitor(listOf(hintCollector, constantCollector)) + visitor.visit(method.ast) + return hintCollector + } +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/engine/fuzzing/FuzzingEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/engine/fuzzing/FuzzingEngine.kt new file mode 100644 index 0000000000..18fbbda640 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/engine/fuzzing/FuzzingEngine.kt @@ -0,0 +1,465 @@ +package org.utbot.python.engine.fuzzing + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.runBlocking +import mu.KotlinLogging +import org.utbot.framework.plugin.api.Coverage +import org.utbot.framework.plugin.api.DocRegularStmt +import org.utbot.framework.plugin.api.EnvironmentModels +import org.utbot.framework.plugin.api.TimeoutException +import org.utbot.framework.plugin.api.UtError +import org.utbot.framework.plugin.api.UtExecutionFailure +import org.utbot.framework.plugin.api.UtExecutionResult +import org.utbot.framework.plugin.api.UtExecutionSuccess +import org.utbot.framework.plugin.api.UtImplicitlyThrownException +import org.utbot.framework.plugin.api.UtTimeoutException +import org.utbot.fuzzing.Control +import org.utbot.fuzzing.NoSeedValueException +import org.utbot.fuzzing.fuzz +import org.utbot.fuzzing.utils.Trie +import org.utbot.python.FunctionArguments +import org.utbot.python.PythonMethod +import org.utbot.python.PythonTestGenerationConfig +import org.utbot.python.coverage.CoverageIdGenerator +import org.utbot.python.coverage.PyInstruction +import org.utbot.python.coverage.buildCoverage +import org.utbot.python.engine.CachedExecutionFeedback +import org.utbot.python.engine.ExecutionFeedback +import org.utbot.python.engine.ExecutionStorage +import org.utbot.python.engine.FakeNodeFeedback +import org.utbot.python.engine.InvalidExecution +import org.utbot.python.engine.TypeErrorFeedback +import org.utbot.python.engine.ValidExecution +import org.utbot.python.engine.fuzzing.typeinference.createMethodAnnotationModifications +import org.utbot.python.engine.utils.transformModelList +import org.utbot.python.evaluation.EvaluationCache +import org.utbot.python.evaluation.PythonCodeSocketExecutor +import org.utbot.python.evaluation.PythonEvaluationError +import org.utbot.python.evaluation.PythonEvaluationSuccess +import org.utbot.python.evaluation.PythonEvaluationTimeout +import org.utbot.python.evaluation.PythonWorkerManager +import org.utbot.python.evaluation.serialization.toPythonTree +import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.PythonTreeModel +import org.utbot.python.framework.api.python.PythonTreeWrapper +import org.utbot.python.framework.api.python.PythonUtExecution +import org.utbot.python.framework.api.python.util.pythonStrClassId +import org.utbot.python.fuzzing.PythonExecutionResult +import org.utbot.python.fuzzing.PythonFeedback +import org.utbot.python.fuzzing.PythonFuzzedConcreteValue +import org.utbot.python.fuzzing.PythonFuzzedValue +import org.utbot.python.fuzzing.PythonFuzzing +import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.newtyping.PythonTypeHintsStorage +import org.utbot.python.newtyping.ast.visitor.constants.ConstantCollector +import org.utbot.python.newtyping.ast.visitor.hints.HintCollector +import org.utbot.python.newtyping.general.FunctionType +import org.utbot.python.newtyping.general.UtType +import org.utbot.python.newtyping.inference.InvalidTypeFeedback +import org.utbot.python.newtyping.inference.SuccessFeedback +import org.utbot.python.newtyping.inference.baseline.BaselineAlgorithm +import org.utbot.python.newtyping.inference.baseline.MethodAndVars +import org.utbot.python.newtyping.mypy.MypyInfoBuild +import org.utbot.python.newtyping.mypy.MypyReportLine +import org.utbot.python.newtyping.mypy.getErrorNumber +import org.utbot.python.newtyping.pythonModules +import org.utbot.python.newtyping.pythonTypeName +import org.utbot.python.newtyping.pythonTypeRepresentation +import org.utbot.python.newtyping.utils.getOffsetLine +import org.utbot.python.utils.ExecutionWithTimoutMode +import org.utbot.python.utils.TestGenerationLimitManager +import org.utbot.python.utils.TimeoutMode +import org.utbot.python.utils.camelToSnakeCase +import org.utbot.python.utils.convertToTime +import org.utbot.summary.fuzzer.names.TestSuggestedInfo +import java.net.ServerSocket +import kotlin.math.min +import kotlin.random.Random + +private val logger = KotlinLogging.logger {} +private const val RANDOM_TYPE_FREQUENCY = 4 +private const val MINIMAL_TIMEOUT_FOR_SUBSTITUTION = 4_000 // ms + +class FuzzingEngine( + val method: PythonMethod, + val configuration: PythonTestGenerationConfig, + val typeStorage: PythonTypeHintsStorage, + val hintCollector: HintCollector, + val constantCollector: ConstantCollector, + val mypyStorage: MypyInfoBuild, + val mypyReport: List, + val until: Long, + val executionStorage: ExecutionStorage, +) { + private val cache = EvaluationCache() + + private val constants: List = constantCollector.result + .mapNotNull { (type, value) -> + if (type.pythonTypeName() == pythonStrClassId.name && value is String) { + // Filter doctests + if (value.contains(">>>")) return@mapNotNull null + } + logger.debug { "Collected constant: ${type.pythonTypeRepresentation()}: $value" } + PythonFuzzedConcreteValue(type, value) + } + + fun start() { + logger.info { "Fuzzing until: ${until.convertToTime()}" } + val modifications = createMethodAnnotationModifications(method, typeStorage) + val now = System.currentTimeMillis() + val filterModifications = modifications + .take(minOf(modifications.size, maxOf(((until - now) / MINIMAL_TIMEOUT_FOR_SUBSTITUTION).toInt(), 1))) + .map { (modifiedMethod, additionalVars) -> + logger.info { "Substitution: ${modifiedMethod.methodSignature()}" } + MethodAndVars(modifiedMethod, additionalVars) + } + generateTests(method, filterModifications, until) + } + + private fun generateTests( + method: PythonMethod, + methodModifications: List, + until: Long, + ) { + val timeoutLimitManager = TestGenerationLimitManager( + TimeoutMode, + until, + ) + val namesInModule = mypyStorage.names + .getOrDefault(configuration.testFileInformation.moduleName, emptyList()) + .map { it.name } + .filter { + it.length < 4 || !it.startsWith("__") || !it.endsWith("__") + } + + val sourceFileContent = configuration.testFileInformation.testedFileContent + val algo = BaselineAlgorithm( + typeStorage, + hintCollector.result, + configuration.pythonPath, + MethodAndVars(method, ""), + methodModifications, + configuration.sysPathDirectories, + configuration.testFileInformation.moduleName, + namesInModule, + getErrorNumber( + mypyReport, + configuration.testFileInformation.testedFilePath, + getOffsetLine(sourceFileContent, method.ast.beginOffset), + getOffsetLine(sourceFileContent, method.ast.endOffset) + ), + mypyStorage.buildRoot.configFile, + randomTypeFrequency = RANDOM_TYPE_FREQUENCY, + dMypyTimeout = configuration.timeoutForRun, + ) + + val fuzzerCancellation = { configuration.isCanceled() || timeoutLimitManager.isCancelled() } + runBlocking { + runFuzzing( + method, + algo, + fuzzerCancellation, + until + ).collect { + executionStorage.saveFuzzingExecution(it) + } + } + } + + private fun runFuzzing( + method: PythonMethod, + typeInferenceAlgorithm: BaselineAlgorithm, + isCancelled: () -> Boolean, + until: Long + ): Flow = flow { + ServerSocket(0).use { serverSocket -> + logger.debug { "Server port: ${serverSocket.localPort}" } + val manager = try { + PythonWorkerManager( + serverSocket, + configuration.pythonPath, + until, + configuration.coverageMeasureMode, + configuration.sendCoverageContinuously, + configuration.doNotGenerateStateAssertions, + ) { + PythonCodeSocketExecutor( + method, + configuration.testFileInformation.moduleName, + configuration.pythonPath, + configuration.sysPathDirectories, + min(configuration.timeoutForRun, until - System.currentTimeMillis()), + it, + ) + } + } catch (_: TimeoutException) { + return@use + } + logger.debug { "Executor manager was created successfully" } + + val initialType = (typeInferenceAlgorithm.expandState() ?: method.methodType) as FunctionType + + val pmd = PythonMethodDescription( + method.name, + constants, + typeStorage, + Trie(PyInstruction::id), + Random(0), + TestGenerationLimitManager(ExecutionWithTimoutMode, until, isRootManager = true), + initialType, + ) + + try { + val parameters = method.methodType.arguments + if (parameters.isEmpty()) { + val result = fuzzingResultHandler(pmd, emptyList(), parameters, manager) + result?.let { + emit(it.executionFeedback) + } + } else { + try { + PythonFuzzing(typeStorage, typeInferenceAlgorithm, isCancelled) { description, arguments -> + if (isCancelled()) { + logger.debug { "Fuzzing process was interrupted" } + manager.disconnect() + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } + if (System.currentTimeMillis() >= until) { + logger.debug { "Fuzzing process was interrupted by timeout" } + manager.disconnect() + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } + + if (arguments.any { PythonTree.containsFakeNode(it.tree) }) { + logger.debug { "FakeNode in Python model" } + description.limitManager.addFakeNodeExecutions() + emit(FakeNodeFeedback) + return@PythonFuzzing PythonFeedback(control = Control.CONTINUE) + } else { + description.limitManager.restartFakeNode() + } + + val pair = Pair(description, arguments.map { PythonTreeWrapper(it.tree) }) + val mem = cache.get(pair) + if (mem != null) { + logger.debug { "Repeat in fuzzing ${arguments.map { it.tree }}" } + description.limitManager.addSuccessExecution() + emit(CachedExecutionFeedback(mem.executionFeedback)) + return@PythonFuzzing mem.fuzzingPlatformFeedback.fromCache() + } + val result = fuzzingResultHandler(description, arguments, parameters, manager) + if (result == null) { // timeout + manager.disconnect() + return@PythonFuzzing PythonFeedback(control = Control.STOP) + } + + cache.add(pair, result) + emit(result.executionFeedback) + return@PythonFuzzing result.fuzzingPlatformFeedback + }.fuzz(pmd) + } catch (_: NoSeedValueException) { + logger.debug { "Cannot fuzz values for types: ${parameters.map { it.pythonTypeRepresentation() }}" } + } + } + } finally { + manager.shutdown() + } + } + }.flowOn(Dispatchers.IO) + + private fun suggestExecutionName( + description: PythonMethodDescription, + executionResult: UtExecutionResult + ): TestSuggestedInfo { + val testSuffix = when (executionResult) { + is UtExecutionSuccess -> { + // can be improved + description.name + } + is UtExecutionFailure -> "${description.name}_with_exception" + else -> description.name + } + val testName = "test_$testSuffix" + return TestSuggestedInfo( + testName, + testName, + ) + } + + private fun handleTimeoutResult( + arguments: List, + methodUnderTestDescription: PythonMethodDescription, + coveredInstructions: List, + ): ExecutionFeedback { + val summary = arguments + .zip(method.arguments) + .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } + val executionResult = UtTimeoutException(TimeoutException("Execution is too long")) + val testMethodName = suggestExecutionName(methodUnderTestDescription, executionResult) + + val hasThisObject = method.hasThisArgument + val (beforeThisObjectTree, beforeModelListTree) = if (hasThisObject) { + arguments.first() to arguments.drop(1) + } else { + null to arguments + } + val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) } + val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) } + + val coverage = Coverage(coveredInstructions) + val utFuzzedExecution = PythonUtExecution( + stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), + stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), + stateAfter = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), + diffIds = emptyList(), + result = executionResult, + coverage = coverage, + testMethodName = testMethodName.testName?.camelToSnakeCase(), + displayName = testMethodName.displayName, + summary = summary.map { DocRegularStmt(it) }, + arguments = method.argumentsWithoutSelf + ) + return ValidExecution(utFuzzedExecution) + } + + private fun handleSuccessResult( + arguments: List, + types: List, + evaluationResult: PythonEvaluationSuccess, + methodUnderTestDescription: PythonMethodDescription, + ): ExecutionFeedback { + val summary = arguments + .zip(method.arguments) + .mapNotNull { it.first.summary?.replace("%var%", it.second.name) } + val hasThisObject = method.hasThisArgument + + val resultModel = evaluationResult.stateAfter.getById(evaluationResult.resultId).toPythonTree(evaluationResult.stateAfter) + + if (evaluationResult.isException && (resultModel.type.name in configuration.prohibitedExceptions)) { // wrong type (sometimes mypy fails) + val errorMessage = "Evaluation with prohibited exception. Substituted types: ${ + types.joinToString { it.pythonTypeRepresentation() } + }. Exception type: ${resultModel.type.name}" + + logger.debug { errorMessage } + return TypeErrorFeedback(errorMessage) + } + + val executionResult = + if (evaluationResult.isException) { + UtImplicitlyThrownException(Throwable(resultModel.type.toString()), false) + } + else { + UtExecutionSuccess(PythonTreeModel(resultModel)) + } + + val testMethodName = suggestExecutionName(methodUnderTestDescription, executionResult) + + val (thisObject, initModelList) = transformModelList(hasThisObject, evaluationResult.stateInit, evaluationResult.modelListIds) + val (beforeThisObject, beforeModelList) = transformModelList(hasThisObject, evaluationResult.stateBefore, evaluationResult.modelListIds) + val (afterThisObject, afterModelList) = transformModelList(hasThisObject, evaluationResult.stateAfter, evaluationResult.modelListIds) + + val utFuzzedExecution = PythonUtExecution( + stateInit = EnvironmentModels(thisObject, initModelList, emptyMap(), executableToCall = null), + stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), + stateAfter = EnvironmentModels(afterThisObject, afterModelList, emptyMap(), executableToCall = null), + diffIds = evaluationResult.diffIds, + result = executionResult, + coverage = buildCoverage(evaluationResult.coveredStatements, evaluationResult.missedStatements), + testMethodName = testMethodName.testName?.camelToSnakeCase(), + displayName = testMethodName.displayName, + summary = summary.map { DocRegularStmt(it) }, + arguments = method.argumentsWithoutSelf, + ) + return ValidExecution(utFuzzedExecution) + } + + private fun fuzzingResultHandler( + description: PythonMethodDescription, + arguments: List, + parameters: List, + manager: PythonWorkerManager, + ): PythonExecutionResult? { + val additionalModules = parameters.flatMap { it.pythonModules() } + + val argumentValues = arguments.map { PythonTreeModel(it.tree, it.tree.type) } + val moduleToImport = configuration.testFileInformation.moduleName + val argumentModules = argumentValues + .flatMap { it.allContainingClassIds } + .map { it.moduleName } + .filterNot { it.startsWith(moduleToImport) } + val localAdditionalModules = (additionalModules + argumentModules + moduleToImport).toSet() + + val (thisObject, modelList) = if (method.hasThisArgument) + Pair(argumentValues[0], argumentValues.drop(1)) + else + Pair(null, argumentValues) + val functionArguments = FunctionArguments( + thisObject, + method.thisObjectName, + modelList, + method.argumentsNamesWithoutSelf + ) + try { + val coverageId = CoverageIdGenerator.createId() + return when (val evaluationResult = + manager.runWithCoverage(functionArguments, localAdditionalModules, coverageId)) { + is PythonEvaluationError -> { + val stackTraceMessage = evaluationResult.stackTrace.joinToString("\n") + val utError = UtError( + "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}\n${stackTraceMessage}", + Throwable(stackTraceMessage) + ) + description.limitManager.addInvalidExecution() + logger.debug(stackTraceMessage) + PythonExecutionResult(InvalidExecution(utError), PythonFeedback(control = Control.PASS)) + } + + is PythonEvaluationTimeout -> { + val utTimeoutException = handleTimeoutResult(arguments, description, emptyList()) + val trieNode: Trie.Node = Trie.emptyNode() + description.limitManager.addInvalidExecution() + PythonExecutionResult( + utTimeoutException, + PythonFeedback(control = Control.PASS, result = trieNode, SuccessFeedback) + ) + } + + is PythonEvaluationSuccess -> { + val coveredInstructions = evaluationResult.coveredStatements + + val result = handleSuccessResult( + arguments, + parameters, + evaluationResult, + description, + ) + val typeInferenceFeedback = if (result is ValidExecution) SuccessFeedback else InvalidTypeFeedback + when (result) { + is ValidExecution -> { + val trieNode: Trie.Node = description.tracer.add(coveredInstructions) + description.limitManager.addSuccessExecution() + PythonExecutionResult( + result, + PythonFeedback(Control.CONTINUE, trieNode, typeInferenceFeedback) + ) + } + is InvalidExecution -> { + description.limitManager.addInvalidExecution() + PythonExecutionResult(result, PythonFeedback(control = Control.CONTINUE, typeInferenceFeedback = typeInferenceFeedback)) + } + else -> { + description.limitManager.addInvalidExecution() + PythonExecutionResult(result, PythonFeedback(control = Control.PASS, typeInferenceFeedback = typeInferenceFeedback)) + } + } + } + } + } catch (_: TimeoutException) { + logger.debug { "Fuzzing process was interrupted by timeout" } + return null + } + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/engine/fuzzing/typeinference/FunctionAnnotationUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/engine/fuzzing/typeinference/FunctionAnnotationUtils.kt new file mode 100644 index 0000000000..64928d2c57 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/engine/fuzzing/typeinference/FunctionAnnotationUtils.kt @@ -0,0 +1,91 @@ +package org.utbot.python.engine.fuzzing.typeinference + +import org.utbot.python.PythonMethod +import org.utbot.python.newtyping.PythonSubtypeChecker +import org.utbot.python.newtyping.PythonTypeHintsStorage +import org.utbot.python.newtyping.PythonTypeVarDescription +import org.utbot.python.newtyping.general.DefaultSubstitutionProvider +import org.utbot.python.newtyping.general.FunctionType +import org.utbot.python.newtyping.general.TypeParameter +import org.utbot.python.newtyping.general.UtType +import org.utbot.python.newtyping.general.getBoundedParameters +import org.utbot.python.newtyping.general.hasBoundedParameters +import org.utbot.python.newtyping.getPythonAttributeByName +import org.utbot.python.newtyping.pythonDescription +import org.utbot.python.newtyping.pythonTypeName +import org.utbot.python.utils.PriorityCartesianProduct + +private const val MAX_SUBSTITUTIONS = 10 +private val BAD_TYPES = setOf( + "builtins.function", + "builtins.super", + "builtins.type", + "builtins.slice", + "builtins.range", + "builtins.memoryview", + "builtins.object", +) + +fun getCandidates(param: TypeParameter, typeStorage: PythonTypeHintsStorage): List { + val meta = param.pythonDescription() as PythonTypeVarDescription + return when (meta.parameterKind) { + PythonTypeVarDescription.ParameterKind.WithConcreteValues -> { + param.constraints.map { it.boundary } + } + + PythonTypeVarDescription.ParameterKind.WithUpperBound -> { + typeStorage.simpleTypes.filter { + if (it.hasBoundedParameters()) + return@filter false + val bound = param.constraints.first().boundary + PythonSubtypeChecker.checkIfRightIsSubtypeOfLeft(bound, it, typeStorage) + } + } + } +} + +fun generateTypesAfterSubstitution(type: UtType, typeStorage: PythonTypeHintsStorage): List { + val params = type.getBoundedParameters() + return PriorityCartesianProduct(params.map { getCandidates(it, typeStorage) }).getSequence() + .filter { types -> types.all { it.pythonTypeName() !in BAD_TYPES } } + .map { subst -> + DefaultSubstitutionProvider.substitute(type, (params zip subst).associate { it }) + }.take(MAX_SUBSTITUTIONS).toList() +} + +fun substituteTypeParameters( + method: PythonMethod, + typeStorage: PythonTypeHintsStorage, +): List { + val newClasses = method.containingPythonClass?.let { + generateTypesAfterSubstitution(it, typeStorage) + } ?: listOf(null) + return newClasses.flatMap { newClass -> + val funcType = newClass?.getPythonAttributeByName(typeStorage, method.name)?.type as? FunctionType + ?: method.methodType + val newFuncTypes = generateTypesAfterSubstitution(funcType, typeStorage) + newFuncTypes.map { newFuncType -> + method.makeCopyWithNewType(newFuncType as FunctionType) + } + }.take(MAX_SUBSTITUTIONS) +} + + +data class ModifiedAnnotation( + val method: PythonMethod, + val additionalVars: String +) + +fun createMethodAnnotationModifications( + method: PythonMethod, + typeStorage: PythonTypeHintsStorage, +): List { + return substituteTypeParameters(method, typeStorage).flatMap { newMethod -> + listOfNotNull( + newMethod.createShortForm(), + (newMethod to "") + ) + }.map { + ModifiedAnnotation(it.first, it.second) + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/SymbolicEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/SymbolicEngine.kt new file mode 100644 index 0000000000..890b2e5dd7 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/SymbolicEngine.kt @@ -0,0 +1,57 @@ +package org.utbot.python.engine.symbolic + +import mu.KotlinLogging +import org.usvm.runner.DebugRunner +import org.usvm.runner.PythonSymbolicAnalysisRunnerImpl +import org.usvm.runner.USVMPythonConfig +import org.usvm.runner.USVMPythonRunConfig +import org.utbot.python.PythonTestGenerationConfig + +private val logger = KotlinLogging.logger {} + +interface SymbolicEngineApi { + fun analyze(runConfig: USVMPythonRunConfig, receiver: USVMPythonAnalysisResultReceiverImpl) +} + +/* +class DummySymbolicEngine( + val configuration: PythonTestGenerationConfig, + val executionStorage: ExecutionStorage, +) : SymbolicEngineApi { + override fun analyze(runConfig: USVMPythonRunConfig, receiver: USVMPythonAnalysisResultReceiverImpl) { + val endTime = System.currentTimeMillis() + runConfig.timeoutMs + val symbolicCancellation = { configuration.isCanceled() || System.currentTimeMillis() > endTime } + while (!symbolicCancellation()) { + receiver.receivePickledInputValuesWithFeedback("""b'\x80\x04\x95\x11\x00\x00\x00\x00\x00\x00\x00K\x02]\x94(K\x01K\x02e\x86\x94}\x94\x86\x94.'""")?.let { + logger.info { "SYMBOLIC: save new execution" } + executionStorage.saveSymbolicExecution(it) + } + Thread.sleep(100) + } + } +} +*/ + +class SymbolicEngine( + val usvmPythonConfig: USVMPythonConfig, + val configuration: PythonTestGenerationConfig, +) : SymbolicEngineApi { + override fun analyze(runConfig: USVMPythonRunConfig, receiver: USVMPythonAnalysisResultReceiverImpl) { + val endTime = System.currentTimeMillis() + runConfig.timeoutMs + val symbolicCancellation = { configuration.isCanceled() || System.currentTimeMillis() > endTime } + val symbolicRunner = PythonSymbolicAnalysisRunnerImpl(usvmPythonConfig) + symbolicRunner.use { + it.analyze(runConfig, receiver, symbolicCancellation) + } + } + + fun debugRun(runConfig: USVMPythonRunConfig) { + val debugRunner = DebugRunner(usvmPythonConfig) + try { + debugRunner.runProcessAndPrintInfo(runConfig) + } catch (ex: Exception) { + logger.error { ex.message } + logger.error { ex.stackTrace } + } + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/SymbolicExecutionEvaluator.kt b/utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/SymbolicExecutionEvaluator.kt new file mode 100644 index 0000000000..6c2632182c --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/SymbolicExecutionEvaluator.kt @@ -0,0 +1,195 @@ +package org.utbot.python.engine.symbolic + +import mu.KotlinLogging +import org.utbot.framework.plugin.api.DocRegularStmt +import org.utbot.framework.plugin.api.EnvironmentModels +import org.utbot.framework.plugin.api.TimeoutException +import org.utbot.framework.plugin.api.UtError +import org.utbot.framework.plugin.api.UtExecutionFailure +import org.utbot.framework.plugin.api.UtExecutionResult +import org.utbot.framework.plugin.api.UtExecutionSuccess +import org.utbot.framework.plugin.api.UtImplicitlyThrownException +import org.utbot.python.PythonMethod +import org.utbot.python.PythonTestGenerationConfig +import org.utbot.python.coverage.CoverageIdGenerator +import org.utbot.python.coverage.buildCoverage +import org.utbot.python.engine.ExecutionFeedback +import org.utbot.python.engine.ExecutionStorage +import org.utbot.python.engine.InvalidExecution +import org.utbot.python.engine.TypeErrorFeedback +import org.utbot.python.engine.ValidExecution +import org.utbot.python.engine.utils.transformModelList +import org.utbot.python.evaluation.PythonCodeSocketExecutor +import org.utbot.python.evaluation.PythonEvaluationError +import org.utbot.python.evaluation.PythonEvaluationSuccess +import org.utbot.python.evaluation.PythonEvaluationTimeout +import org.utbot.python.evaluation.PythonWorkerManager +import org.utbot.python.evaluation.serialization.toPythonTree +import org.utbot.python.framework.api.python.PythonSymbolicUtExecution +import org.utbot.python.framework.api.python.PythonTreeModel +import org.utbot.python.utils.camelToSnakeCase +import org.utbot.summary.fuzzer.names.TestSuggestedInfo +import java.net.ServerSocket +import java.net.SocketException +import java.util.* +import kotlin.math.min + +private val logger = KotlinLogging.logger {} + +class SymbolicExecutionEvaluator( + val method: PythonMethod, + val configuration: PythonTestGenerationConfig, + val executionStorage: ExecutionStorage, + val until: Long, + var receiverFinished: Boolean = false +): Thread() { + private lateinit var serverSocket: ServerSocket + private lateinit var manager: PythonWorkerManager + val executionQueue: MutableList = Collections.synchronizedList(mutableListOf()) + + init { + connect() + } + + private fun connect() { + try { + serverSocket = ServerSocket(0) + manager = + PythonWorkerManager( + serverSocket, + configuration.pythonPath, + until, + configuration.coverageMeasureMode, + configuration.sendCoverageContinuously, + configuration.doNotGenerateStateAssertions, + ) { + PythonCodeSocketExecutor( + method, + configuration.testFileInformation.moduleName, + configuration.pythonPath, + configuration.sysPathDirectories, + min(configuration.timeoutForRun, until - System.currentTimeMillis()), + it, + ) + } + } catch (_: TimeoutException) { + close() + } + } + + + private fun receivePickledInputValuesWithFeedback(pickledTuple: String): ExecutionFeedback? { + if (System.currentTimeMillis() >= until) + return null + try { + logger.debug("SYMBOLIC running $pickledTuple") + val coverageId = CoverageIdGenerator.createId() + return when ( + val evaluationResult = manager.runWithCoverage(pickledTuple, coverageId) + ) { + is PythonEvaluationError -> { + val stackTraceMessage = evaluationResult.stackTrace.joinToString("\n") + val utError = UtError( + "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}\n${stackTraceMessage}", + Throwable(stackTraceMessage) + ) + logger.debug(stackTraceMessage) + InvalidExecution(utError) + } + + is PythonEvaluationTimeout -> { +// val coveredInstructions = manager.coverageReceiver.coverageStorage.getOrDefault(coverageId, mutableListOf()) +// handleTimeoutResult(method, coveredInstructions) + null + } + + is PythonEvaluationSuccess -> { + handleSuccessResult(method, evaluationResult) + } + } + } catch (_: TimeoutException) { + logger.debug { "Symbolic process was interrupted by timeout" } + return null + } catch (_: SocketException) { + connect() + return null + } + } + + private fun suggestExecutionName( + method: PythonMethod, + executionResult: UtExecutionResult + ): TestSuggestedInfo { + val testSuffix = when (executionResult) { + is UtExecutionSuccess -> { + // can be improved + method.name + } + is UtExecutionFailure -> "${method.name}_with_exception" + else -> method.name + } + val testName = "test_$testSuffix" + return TestSuggestedInfo( + testName, + testName, + ) + } + + private fun handleSuccessResult( + method: PythonMethod, + evaluationResult: PythonEvaluationSuccess + ): ExecutionFeedback { + val summary = emptyList() // TODO: improve + val hasThisObject = method.hasThisArgument + val resultModel = evaluationResult.stateAfter.getById(evaluationResult.resultId).toPythonTree(evaluationResult.stateAfter) + + if (evaluationResult.isException && (resultModel.type.name in configuration.prohibitedExceptions)) { // wrong type (sometimes mypy fails) + val errorMessage = "Evaluation with prohibited exception. Error: $resultModel" + logger.debug { errorMessage } + return TypeErrorFeedback(errorMessage) + } + val executionResult = + if (evaluationResult.isException) { + UtImplicitlyThrownException(Throwable(resultModel.type.toString()), false) + } + else { + UtExecutionSuccess(PythonTreeModel(resultModel)) + } + val testMethodName = suggestExecutionName(method, executionResult) + + val (beforeThisObject, beforeModelList) = transformModelList(hasThisObject, evaluationResult.stateBefore, evaluationResult.modelListIds) + val (afterThisObject, afterModelList) = transformModelList(hasThisObject, evaluationResult.stateAfter, evaluationResult.modelListIds) + + val utFuzzedExecution = PythonSymbolicUtExecution( + stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap(), executableToCall = null), + stateAfter = EnvironmentModels(afterThisObject, afterModelList, emptyMap(), executableToCall = null), + diffIds = evaluationResult.diffIds, + result = executionResult, + coverage = buildCoverage(evaluationResult.coveredStatements, evaluationResult.missedStatements), + testMethodName = testMethodName.testName?.camelToSnakeCase(), + displayName = testMethodName.displayName, + summary = summary.map { DocRegularStmt(it) }, + arguments = method.argumentsWithoutSelf, + ) + return ValidExecution(utFuzzedExecution) + } + + fun close() { + manager.shutdown() + serverSocket.close() + } + + override fun run() { + while (System.currentTimeMillis() < until && (!receiverFinished || executionQueue.isNotEmpty())) { + val pickledTuple = if (executionQueue.isNotEmpty()) { + executionQueue.removeFirst() + } else { + sleep(10) + continue + } + receivePickledInputValuesWithFeedback(pickledTuple)?.let { + executionStorage.saveSymbolicExecution(it) + } + } + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/USVMPythonAnalysisResultReceiverImpl.kt b/utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/USVMPythonAnalysisResultReceiverImpl.kt new file mode 100644 index 0000000000..8b88731e40 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/engine/symbolic/USVMPythonAnalysisResultReceiverImpl.kt @@ -0,0 +1,16 @@ +package org.utbot.python.engine.symbolic + +import mu.KotlinLogging +import org.usvm.runner.USVMPythonAnalysisResultReceiver + +private val logger = KotlinLogging.logger {} + +class USVMPythonAnalysisResultReceiverImpl( + private val threadSafeQueue: MutableList +) : USVMPythonAnalysisResultReceiver() { + + override fun receivePickledInputValues(pickledTuple: String) { + logger.debug { "SYMBOLIC: $pickledTuple" } + threadSafeQueue.add(pickledTuple) + } +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/engine/utils/ModelsTransformation.kt b/utbot-python/src/main/kotlin/org/utbot/python/engine/utils/ModelsTransformation.kt new file mode 100644 index 0000000000..cef51ab953 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/engine/utils/ModelsTransformation.kt @@ -0,0 +1,31 @@ +package org.utbot.python.engine.utils + +import org.utbot.framework.plugin.api.UtModel +import org.utbot.python.evaluation.serialization.MemoryDump +import org.utbot.python.evaluation.serialization.toPythonTree +import org.utbot.python.framework.api.python.PythonTreeModel + +fun transformModelList( + hasThisObject: Boolean, + state: MemoryDump, + modelListIds: List +): Pair> { + val (stateThisId, resultModelListIds) = + if (hasThisObject) { + Pair(modelListIds.first(), modelListIds.drop(1)) + } else { + Pair(null, modelListIds) + } + val stateThisObject = stateThisId?.let { + PythonTreeModel( + state.getById(it).toPythonTree(state) + ) + } + val modelList = resultModelListIds.map { + PythonTreeModel( + state.getById(it).toPythonTree(state) + ) + } + return Pair(stateThisObject, modelList) +} + diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt index 7e6eed438a..c7cefa86e3 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt @@ -23,6 +23,11 @@ interface PythonCodeExecutor { coverageId: String, ): PythonEvaluationResult + fun runWithCoverage( + pickledArguments: String, + coverageId: String, + ): PythonEvaluationResult + fun stop() } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index f30872204a..d042b2f979 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -12,6 +12,7 @@ import org.utbot.python.evaluation.serialization.SuccessExecution import org.utbot.python.evaluation.serialization.serializeObjects import org.utbot.python.coverage.CoverageIdGenerator import org.utbot.python.coverage.toPyInstruction +import org.utbot.python.evaluation.serialization.MemoryMode import org.utbot.python.newtyping.PythonCallableTypeDescription import org.utbot.python.newtyping.pythonDescription import org.utbot.python.newtyping.pythonTypeName @@ -59,7 +60,7 @@ class PythonCodeSocketExecutor( ): PythonEvaluationResult { val (arguments, memory) = serializeObjects(fuzzedValues.allArguments.map { it.tree }) - val meta = method.definition.type.pythonDescription() as PythonCallableTypeDescription + val meta = method.methodType.pythonDescription() as PythonCallableTypeDescription val argKinds = meta.argumentKinds val namedArgs = meta.argumentNames .filterIndexed { index, _ -> !isNamed(argKinds[index]) } @@ -87,6 +88,7 @@ class PythonCodeSocketExecutor( positionalArguments.map { it.first }, namedArguments.associate { it.second!! to it.first }, // here can be only-kwargs arguments memory, + MemoryMode.REDUCE, method.moduleFilename, coverageId, ) @@ -115,6 +117,53 @@ class PythonCodeSocketExecutor( } } + override fun runWithCoverage(pickledArguments: String, coverageId: String): PythonEvaluationResult { + val containingClass = method.containingPythonClass + val functionTextName = + if (containingClass == null) + method.name + else { + val fullname = "${containingClass.pythonTypeName()}.${method.name}" + fullname.drop(moduleToImport.length).removePrefix(".") + } + + val request = ExecutionRequest( + functionTextName, + moduleToImport, + emptyList(), + syspathDirectories.toList(), + emptyList(), + emptyMap(), + pickledArguments, + MemoryMode.PICKLE, + method.moduleFilename, + coverageId, + ) + val message = ExecutionRequestSerializer.serializeRequest(request) ?: error("Cannot serialize request to python executor") + try { + pythonWorker.sendData(message) + } catch (_: SocketException) { + return parseExecutionResult(FailExecution("Send data error")) + } + + val (status, response) = UtExecutorThread.run(pythonWorker, executionTimeout) + + return when (status) { + UtExecutorThread.Status.TIMEOUT -> { + PythonEvaluationTimeout() + } + + UtExecutorThread.Status.OK -> { + val executionResult = response?.let { + ExecutionResultDeserializer.parseExecutionResult(it) + ?: error("Cannot parse execution result: $it") + } ?: FailExecution("Execution result error") + + parseExecutionResult(executionResult) + } + } + } + private fun parseExecutionResult(executionResult: PythonExecutionResult): PythonEvaluationResult { val parsingException = PythonEvaluationError( -1, @@ -123,9 +172,9 @@ class PythonCodeSocketExecutor( ) return when (executionResult) { is SuccessExecution -> { - val stateInit = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateInit) ?: return parsingException val stateBefore = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateBefore) ?: return parsingException val stateAfter = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateAfter) ?: return parsingException + val stateInit = ExecutionResultDeserializer.parseMemoryDump(executionResult.stateInit) ?: stateBefore val diffIds = executionResult.diffIds.map {it.toLong()} val statements = executionResult.statements.mapNotNull { it.toPyInstruction() } val missedStatements = executionResult.missedStatements.mapNotNull { it.toPyInstruction() } @@ -141,15 +190,17 @@ class PythonCodeSocketExecutor( executionResult.resultId, ) } - is FailExecution -> PythonEvaluationError( - -2, - "Fail Execution", - executionResult.exception.split(System.lineSeparator()), - ) + is FailExecution -> { + PythonEvaluationError( + -2, + "Fail Execution", + executionResult.exception.split(System.lineSeparator()), + ) + } } } override fun stop() { pythonWorker.stopServer() } -} \ No newline at end of file +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index 0841e1c36a..d63d4ff79f 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -1,17 +1,18 @@ package org.utbot.python.evaluation import mu.KotlinLogging +import org.apache.logging.log4j.LogManager import org.utbot.framework.plugin.api.TimeoutException import org.utbot.python.FunctionArguments +import org.utbot.python.coverage.PythonCoverageMode import org.utbot.python.utils.TemporaryFileManager import org.utbot.python.utils.getResult import org.utbot.python.utils.startProcess import java.lang.Long.max import java.net.ServerSocket import java.net.Socket +import java.net.SocketException import java.net.SocketTimeoutException -import org.apache.logging.log4j.LogManager -import org.utbot.python.coverage.PythonCoverageMode private val logger = KotlinLogging.logger {} @@ -21,6 +22,7 @@ class PythonWorkerManager( val until: Long, private val coverageMeasureMode: PythonCoverageMode = PythonCoverageMode.Instructions, private val sendCoverageContinuously: Boolean = true, + private val doNotGenerateStateAssertions: Boolean, val pythonCodeExecutorConstructor: (PythonWorker) -> PythonCodeExecutor, ) { var timeout: Long = 0 @@ -41,18 +43,24 @@ class PythonWorkerManager( process.destroy() } val logLevel = LogManager.getRootLogger().level.name() - process = startProcess(listOf( - pythonPath, - "-m", "utbot_executor", - "localhost", - serverSocket.localPort.toString(), - coverageReceiver.address().first, - coverageReceiver.address().second, - "--logfile", logfile.absolutePath, - "--loglevel", logLevel, // "DEBUG", "INFO", "WARNING", "ERROR" - "--coverage_type", coverageMeasureMode.toString(), // "lines", "instructions" - sendCoverageContinuously.toSendCoverageContinuouslyString(), // "--send_coverage", "--no-send_coverage" - )) + if (serverSocket.isClosed) { + serverSocket.accept() + } + process = startProcess( + listOf( + pythonPath, + "-m", "utbot_executor", + "localhost", + serverSocket.localPort.toString(), + coverageReceiver.address().first, + coverageReceiver.address().second, + "--logfile", logfile.absolutePath, + "--loglevel", logLevel, // "DEBUG", "INFO", "WARNING", "ERROR" + "--coverage_type", coverageMeasureMode.toString(), // "lines", "instructions" + sendCoverageContinuously.toSendCoverageContinuouslyString(), // "--send_coverage", "--no-send_coverage" + doNotGenerateStateAssertions.toGenerateStateAssertionsString() // "--generate_state_assertions", "--no-generate_state_assertions" + ) + ) timeout = max(until - processStartTime, 0) if (this::workerSocket.isInitialized && !workerSocket.isClosed) { workerSocket.close() @@ -65,6 +73,9 @@ class PythonWorkerManager( logger.debug { "utbot_executor exit value: ${result.exitValue}. stderr: ${result.stderr}, stdout: ${result.stdout}." } process.destroy() throw TimeoutException("Worker not connected") + } catch (e: SocketException) { + logger.debug { e.message } + throw SocketException("Worker not connected: $e") } logger.debug { "Worker connected successfully" } @@ -75,7 +86,7 @@ class PythonWorkerManager( fun disconnect() { workerSocket.close() - process.destroy() + process.destroyForcibly() } private fun reconnect() { @@ -106,6 +117,23 @@ class PythonWorkerManager( return evaluationResult } + fun runWithCoverage( + pickledArguments: String, + coverageId: String + ): PythonEvaluationResult { + val evaluationResult = try { + codeExecutor.runWithCoverage(pickledArguments, coverageId) + } catch (_: SocketTimeoutException) { + logger.debug { "Socket timeout" } + reconnect() + PythonEvaluationTimeout() + } + if (evaluationResult is PythonEvaluationError || evaluationResult is PythonEvaluationTimeout) { + reconnect() + } + return evaluationResult + } + fun run( fuzzedValues: FunctionArguments, additionalModulesToImport: Set @@ -133,5 +161,13 @@ class PythonWorkerManager( "--no-send_coverage" } } + + fun Boolean.toGenerateStateAssertionsString(): String { + return if (this) { + "--no-generate_state_assertions" + } else { + "--generate_state_assertions" + } + } } -} \ No newline at end of file +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/UtExecutorThread.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/UtExecutorThread.kt index 5cbaa96914..fef2c59715 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/UtExecutorThread.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/UtExecutorThread.kt @@ -1,11 +1,13 @@ package org.utbot.python.evaluation +import mu.KotlinLogging import java.net.SocketException import java.util.concurrent.Callable import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException +private val logger = KotlinLogging.logger {} class UtExecutorThread { enum class Status { @@ -18,12 +20,17 @@ class UtExecutorThread { val executor = Executors.newSingleThreadExecutor() val future = executor.submit(Task(worker)) + logger.debug("Running with timeout $executionTimeout") + val result = try { Status.OK to future.get(executionTimeout, TimeUnit.MILLISECONDS) } catch (ex: TimeoutException) { future.cancel(true) Status.TIMEOUT to null } + + logger.debug("Stopped running. Result: {}", result.first) + executor.shutdown() return result } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionRequestSerializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionRequestSerializer.kt index d4b9d7bbc6..a8db047a6a 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionRequestSerializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionRequestSerializer.kt @@ -15,6 +15,11 @@ object ExecutionRequestSerializer { } } +enum class MemoryMode { + PICKLE, + REDUCE +} + data class ExecutionRequest( val functionName: String, val functionModule: String, @@ -23,6 +28,7 @@ data class ExecutionRequest( val argumentsIds: List, val kwargumentsIds: Map, val serializedMemory: String, + val memoryMode: MemoryMode, val filepath: String, val coverageId: String, ) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt index 9cc140a12a..b37905b7f8 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/ExecutionResultDeserializer.kt @@ -18,6 +18,7 @@ object ExecutionResultDeserializer { .withSubtype(ListMemoryObject::class.java, "list") .withSubtype(DictMemoryObject::class.java, "dict") .withSubtype(ReduceMemoryObject::class.java, "reduce") + .withSubtype(IteratorMemoryObject::class.java, "iterator") ) .addLast(KotlinJsonAdapterFactory()) .build() @@ -35,7 +36,11 @@ object ExecutionResultDeserializer { } fun parseMemoryDump(content: String): MemoryDump? { - return jsonAdapterMemoryDump.fromJson(content) + return if (content.isNotEmpty()) { + jsonAdapterMemoryDump.fromJson(content) + } else { + null + } } } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/PythonObjectParser.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/PythonObjectParser.kt index ae1d1aa253..cdcd366fba 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/PythonObjectParser.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialization/PythonObjectParser.kt @@ -14,6 +14,7 @@ object PythonObjectParser { .withSubtype(ListMemoryObject::class.java, "list") .withSubtype(DictMemoryObject::class.java, "dict") .withSubtype(ReduceMemoryObject::class.java, "reduce") + .withSubtype(IteratorMemoryObject::class.java, "iterator") ) .addLast(KotlinJsonAdapterFactory()) .build() @@ -81,6 +82,14 @@ class DictMemoryObject( val items: Map, ) : MemoryObject(id, typeinfo, comparable) +class IteratorMemoryObject( + id: String, + typeinfo: TypeInfo, + comparable: Boolean, + val items: List, + val exception: TypeInfo, +) : MemoryObject(id, typeinfo, comparable) + class ReduceMemoryObject( id: String, typeinfo: TypeInfo, @@ -103,7 +112,7 @@ fun PythonTree.PythonTreeNode.toMemoryObject(memoryDump: MemoryDump, reload: Boo id, typeinfo, this.comparable, - this.repr.replace("\n", "\\\n").replace("\r", "\\\r") + this.repr.replace("\n", "\\n").replace("\r", "\\r") ) } @@ -138,6 +147,12 @@ fun PythonTree.PythonTreeNode.toMemoryObject(memoryDump: MemoryDump, reload: Boo DictMemoryObject(id, typeinfo, this.comparable, items) } + is PythonTree.IteratorNode -> { + val items = this.items.entries + .map { it.value.toMemoryObject(memoryDump) } + IteratorMemoryObject(id, typeinfo, this.comparable, items, TypeInfo(this.exception.moduleName, this.exception.typeName)) + } + is PythonTree.ReduceNode -> { val argsIds = PythonTree.ListNode(this.args.withIndex().associate { it.index to it.value }.toMutableMap()) val draft = ReduceMemoryObject( @@ -236,6 +251,18 @@ fun MemoryObject.toPythonTree( draft } + is IteratorMemoryObject -> { + val draft = PythonTree.IteratorNode(id, mutableMapOf(), PythonClassId(exception.module, exception.kind)) + + visited[this.id] = draft + + items.mapIndexed { index, valueId -> + val value = memoryDump.getById(valueId).toPythonTree(memoryDump, visited) + draft.items[index] = value + } + draft + } + is ReduceMemoryObject -> { val stateObjsDraft = memoryDump.getById(state) val customState = stateObjsDraft !is DictMemoryObject diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt index e2c6ca43ba..2e7d37ca48 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonApi.kt @@ -87,7 +87,7 @@ class PythonTreeModel( } } -class PythonUtExecution( +open class PythonUtExecution( val stateInit: EnvironmentModels, stateBefore: EnvironmentModels, stateAfter: EnvironmentModels, @@ -133,4 +133,16 @@ class PythonUtExecution( arguments = arguments ) } -} \ No newline at end of file +} + +class PythonSymbolicUtExecution( + stateBefore: EnvironmentModels, + stateAfter: EnvironmentModels, + diffIds: List, + result: UtExecutionResult, + arguments: List, + coverage: Coverage? = null, + summary: List? = null, + testMethodName: String? = null, + displayName: String? = null, +) : PythonUtExecution(stateBefore, stateBefore, stateAfter, diffIds, result, arguments, coverage, summary, testMethodName, displayName) \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt index 794c502127..489292fe87 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/PythonTree.kt @@ -4,10 +4,12 @@ import org.utbot.python.framework.api.python.util.pythonBoolClassId import org.utbot.python.framework.api.python.util.pythonDictClassId import org.utbot.python.framework.api.python.util.pythonFloatClassId import org.utbot.python.framework.api.python.util.pythonIntClassId +import org.utbot.python.framework.api.python.util.pythonIteratorClassId import org.utbot.python.framework.api.python.util.pythonListClassId import org.utbot.python.framework.api.python.util.pythonNoneClassId import org.utbot.python.framework.api.python.util.pythonObjectClassId import org.utbot.python.framework.api.python.util.pythonSetClassId +import org.utbot.python.framework.api.python.util.pythonStopIterationClassId import org.utbot.python.framework.api.python.util.pythonStrClassId import org.utbot.python.framework.api.python.util.pythonTupleClassId import org.utbot.python.framework.api.python.util.toPythonRepr @@ -20,18 +22,20 @@ import java.util.concurrent.atomic.AtomicLong object PythonTree { + + val MAX_ITERATOR_SIZE = 1000 + fun isRecursiveObject(tree: PythonTreeNode): Boolean { return isRecursiveObjectDFS(tree, mutableSetOf()) } - private fun isRecursiveObjectDFS(tree: PythonTreeNode, visited: MutableSet): Boolean { + private fun isRecursiveObjectDFS(tree: PythonTreeNode, visited: Set): Boolean { if (tree is PrimitiveNode) { return false } if (visited.contains(tree)) return true - visited.add(tree) - return tree.children.any { isRecursiveObjectDFS(it, visited) } + return tree.children.any { isRecursiveObjectDFS(it, visited + tree) } } fun containsFakeNode(tree: PythonTreeNode): Boolean { @@ -213,6 +217,26 @@ object PythonTree { } } + class IteratorNode( + id: Long, + val items: MutableMap, + val exception: PythonClassId = pythonStopIterationClassId, + ) : PythonTreeNode(id, pythonIteratorClassId) { + + constructor(items: MutableMap, stopException: PythonClassId = pythonStopIterationClassId) : this(PythonIdGenerator.createId(), items, stopException) + + override val children: List + get() = items.values.toList() + + override fun typeEquals(other: Any?): Boolean { + return if (other is ListNode) + children.zip(other.children).all { + it.first.typeEquals(it.second) + } + else false + } + } + class ReduceNode( id: Long, type: PythonClassId, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt index 5cd07a09f5..01d3b66e82 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonIdUtils.kt @@ -20,4 +20,6 @@ val pythonSetClassId = PythonClassId("builtins.set") val pythonBytearrayClassId = PythonClassId("builtins.bytearray") val pythonBytesClassId = PythonClassId("builtins.bytes") val pythonExceptionClassId = PythonClassId("builtins.Exception") +val pythonIteratorClassId = PythonClassId("typing.Iterator") val pythonRePatternClassId = PythonClassId("re.Pattern") +val pythonStopIterationClassId = PythonClassId("builtins.StopIteration") diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonUtils.kt index d4237e9614..fc7fccf1b1 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/api/python/util/PythonUtils.kt @@ -13,7 +13,9 @@ fun String.toSnakeCase(): String { else if (c.isUpperCase()) { (if (index > 0) "_" else "") + c.lowercase() } else c - }.joinToString("") + } + .joinToString("") + .replace(".", "") } fun String.toPythonRepr(): String { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonCodeGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonCodeGenerator.kt index 454ddac11c..6df2bcab87 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonCodeGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/PythonCodeGenerator.kt @@ -106,7 +106,7 @@ class PythonCodeGenerator( imports.forEach { renderer.renderPythonImport(it) } - val paramNames = method.definition.meta.args.map { it.name } + val paramNames = method.argumentsNames val parameters = paramNames.map { argument -> "${argument}: ${methodAnnotations[argument]?.pythonTypeRepresentation() ?: pythonAnyType.pythonTypeRepresentation()}" } @@ -121,7 +121,7 @@ class PythonCodeGenerator( additionalVars, "", functionName, - ) + method.codeAsString.split("\n").map { " $it" } + ) + method.codeAsString.split("\n") return mypyCheckCode.joinToString("\n") } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgMethodConstructor.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgMethodConstructor.kt index d31cf52346..1fa576e4bc 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgMethodConstructor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgMethodConstructor.kt @@ -5,7 +5,6 @@ import org.utbot.framework.codegen.domain.context.CgContext import org.utbot.framework.codegen.domain.models.CgDocumentationComment import org.utbot.framework.codegen.domain.models.CgFieldAccess import org.utbot.framework.codegen.domain.models.CgGetLength -import org.utbot.framework.codegen.domain.models.CgLiteral import org.utbot.framework.codegen.domain.models.CgMethodTestSet import org.utbot.framework.codegen.domain.models.CgMultilineComment import org.utbot.framework.codegen.domain.models.CgParameterDeclaration @@ -17,14 +16,32 @@ import org.utbot.framework.codegen.domain.models.CgVariable import org.utbot.framework.codegen.domain.models.convertDocToCg import org.utbot.framework.codegen.tree.CgMethodConstructor import org.utbot.framework.codegen.tree.buildTestMethod -import org.utbot.framework.plugin.api.* -import org.utbot.python.framework.api.python.* +import org.utbot.framework.plugin.api.ExecutableId +import org.utbot.framework.plugin.api.FieldId +import org.utbot.framework.plugin.api.InstrumentedProcessDeathException +import org.utbot.framework.plugin.api.MethodId +import org.utbot.framework.plugin.api.TimeoutException +import org.utbot.framework.plugin.api.UtExecution +import org.utbot.framework.plugin.api.UtExecutionFailure +import org.utbot.framework.plugin.api.UtModel +import org.utbot.framework.plugin.api.UtTimeoutException +import org.utbot.python.framework.api.python.PythonClassId +import org.utbot.python.framework.api.python.PythonMethodId +import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.PythonTreeModel +import org.utbot.python.framework.api.python.PythonUtExecution import org.utbot.python.framework.api.python.util.pythonExceptionClassId import org.utbot.python.framework.api.python.util.pythonIntClassId import org.utbot.python.framework.api.python.util.pythonNoneClassId import org.utbot.python.framework.codegen.PythonCgLanguageAssistant import org.utbot.python.framework.codegen.model.constructor.util.importIfNeeded -import org.utbot.python.framework.codegen.model.tree.* +import org.utbot.python.framework.codegen.model.tree.CgPythonFunctionCall +import org.utbot.python.framework.codegen.model.tree.CgPythonIndex +import org.utbot.python.framework.codegen.model.tree.CgPythonNamedArgument +import org.utbot.python.framework.codegen.model.tree.CgPythonRange +import org.utbot.python.framework.codegen.model.tree.CgPythonRepr +import org.utbot.python.framework.codegen.model.tree.CgPythonTree +import org.utbot.python.framework.codegen.model.tree.CgPythonZip class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(context) { private val maxDepth: Int = 5 @@ -339,6 +356,49 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex pythonAssertElementsByKey(expectedNode, expectedCollection, actual, iterator, elementName) } + private fun pythonAssertIterators( + expectedNode: PythonTree.IteratorNode, + expected: CgValue, + actual: CgVariable, + ) { + val zip = CgPythonZip( + variableConstructor.getOrCreateVariable(PythonTreeModel(expectedNode)), + actual, + ) + val index = newVar(pythonNoneClassId, "pair") { + CgPythonRepr(pythonNoneClassId, "None") + } + forEachLoop { + innerBlock { + condition = index + iterable = zip + testFrameworkManager.assertEquals( + CgPythonIndex( + pythonIntClassId, + index, + CgPythonRepr(pythonIntClassId, "1") + ), + CgPythonIndex( + pythonIntClassId, + index, + CgPythonRepr(pythonIntClassId, "0") + ) + ) + statements = currentBlock + } + } + if (expectedNode.items.size < PythonTree.MAX_ITERATOR_SIZE) { + testFrameworkManager.expectException(expectedNode.exception) { + +CgPythonFunctionCall( + PythonClassId("builtins.next"), + "next", + listOf(actual) + ) + emptyLineIfNeeded() + } + } + } + private fun pythonDeepTreeEquals( expectedNode: PythonTree.PythonTreeNode, expected: CgValue, @@ -358,10 +418,18 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex } else { variableConstructor.getOrCreateVariable(PythonTreeModel(expectedNode)) } - testFrameworkManager.assertEquals( - expectedValue, - actual, - ) + if (expectedNode is PythonTree.IteratorNode) { + pythonAssertIterators( + expectedNode, + expected, + actual + ) + } else { + testFrameworkManager.assertEquals( + expectedValue, + actual, + ) + } return } when (expectedNode) { @@ -405,6 +473,14 @@ class PythonCgMethodConstructor(context: CgContext) : CgMethodConstructor(contex ) } + is PythonTree.IteratorNode -> { + pythonAssertIterators( + expectedNode, + expected, + actual + ) + } + is PythonTree.ReduceNode -> { if (expectedNode.state.isNotEmpty()) { expectedNode.state.forEach { (field, value) -> diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt index 6c9369c619..715285ccf1 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonCgVariableConstructor.kt @@ -10,13 +10,26 @@ import org.utbot.framework.codegen.tree.CgVariableConstructor import org.utbot.framework.plugin.api.ConstructorId import org.utbot.framework.plugin.api.FieldId import org.utbot.framework.plugin.api.UtModel -import org.utbot.python.framework.api.python.* +import org.utbot.python.framework.api.python.NormalizedPythonAnnotation +import org.utbot.python.framework.api.python.PythonClassId +import org.utbot.python.framework.api.python.PythonMethodId +import org.utbot.python.framework.api.python.PythonModel +import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.PythonTreeModel +import org.utbot.python.framework.api.python.RawPythonAnnotation import org.utbot.python.framework.api.python.util.comparePythonTree import org.utbot.python.framework.api.python.util.pythonDictClassId import org.utbot.python.framework.api.python.util.pythonListClassId import org.utbot.python.framework.api.python.util.pythonNoneClassId import org.utbot.python.framework.codegen.PythonCgLanguageAssistant -import org.utbot.python.framework.codegen.model.tree.* +import org.utbot.python.framework.codegen.model.tree.CgPythonDict +import org.utbot.python.framework.codegen.model.tree.CgPythonIndex +import org.utbot.python.framework.codegen.model.tree.CgPythonIterator +import org.utbot.python.framework.codegen.model.tree.CgPythonList +import org.utbot.python.framework.codegen.model.tree.CgPythonRepr +import org.utbot.python.framework.codegen.model.tree.CgPythonSet +import org.utbot.python.framework.codegen.model.tree.CgPythonTree +import org.utbot.python.framework.codegen.model.tree.CgPythonTuple class PythonCgVariableConstructor(cgContext: CgContext) : CgVariableConstructor(cgContext) { private val nameGenerator = CgComponents.getNameGeneratorBy(context) @@ -97,6 +110,11 @@ class PythonCgVariableConstructor(cgContext: CgContext) : CgVariableConstructor( } } + is PythonTree.IteratorNode -> { + val items = objectNode.items.values.map { pythonBuildObject(it) } + Pair(CgPythonIterator(items.map {it.first}, objectNode.exception), items.flatMap { it.second }) + } + is PythonTree.ReduceNode -> { if (assistant.memoryObjects.containsKey(id)) { val tree = assistant.memoryObjectsModels[id] diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonTestFrameworkManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonTestFrameworkManager.kt index cce442c13a..2bd9e38bb2 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonTestFrameworkManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/tree/PythonTestFrameworkManager.kt @@ -1,9 +1,13 @@ package org.utbot.python.framework.codegen.model.constructor.tree -import org.utbot.framework.codegen.domain.context.TestClassContext import org.utbot.framework.codegen.domain.context.CgContext -import org.utbot.framework.codegen.domain.models.* -import org.utbot.framework.codegen.domain.models.AnnotationTarget.* +import org.utbot.framework.codegen.domain.context.TestClassContext +import org.utbot.framework.codegen.domain.models.AnnotationTarget.Method +import org.utbot.framework.codegen.domain.models.CgEqualTo +import org.utbot.framework.codegen.domain.models.CgLiteral +import org.utbot.framework.codegen.domain.models.CgNamedAnnotationArgument +import org.utbot.framework.codegen.domain.models.CgValue +import org.utbot.framework.codegen.domain.models.CgVariable import org.utbot.framework.codegen.services.framework.TestFrameworkManager import org.utbot.framework.plugin.api.ClassId import org.utbot.python.framework.api.python.PythonClassId @@ -25,6 +29,7 @@ internal class PytestManager(context: CgContext) : TestFrameworkManager(context) require(testFramework is Pytest) { "According to settings, Pytest was expected, but got: $testFramework" } require(exception is PythonClassId) { "Exceptions must be PythonClassId" } context.importIfNeeded(PythonClassId("pytest.raises")) + importIfNeeded(exception) val withExpression = CgPythonFunctionCall( pythonNoneClassId, "pytest.raises", @@ -119,6 +124,7 @@ internal class UnittestManager(context: CgContext) : TestFrameworkManager(contex override fun expectException(exception: ClassId, block: () -> Unit) { require(testFramework is Unittest) { "According to settings, Unittest was expected, but got: $testFramework" } require(exception is PythonClassId) { "Exceptions must be PythonClassId" } + importIfNeeded(exception) val withExpression = CgPythonFunctionCall( pythonNoneClassId, "self.assertRaises", diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonRenderer.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonRenderer.kt index cc7e868cce..efd00881a6 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonRenderer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonRenderer.kt @@ -1,8 +1,6 @@ package org.utbot.python.framework.codegen.model.constructor.visitor import org.apache.commons.text.StringEscapeUtils -import org.utbot.common.WorkaroundReason -import org.utbot.common.workaround import org.utbot.python.framework.codegen.model.PythonImport import org.utbot.python.framework.codegen.model.PythonSysPathImport import org.utbot.framework.codegen.domain.RegularImport @@ -537,6 +535,12 @@ internal class CgPythonRenderer( print(")") } + override fun visit(element: CgPythonZip) { + print("zip(") + listOf(element.first, element.second).renderSeparated() + print(")") + } + override fun visit(element: CgPythonList) { print("[") element.elements.renderSeparated() @@ -566,6 +570,12 @@ internal class CgPythonRenderer( } } + override fun visit(element: CgPythonIterator) { + print("iter([") + element.elements.renderSeparated() + print("])") + } + override fun visit(element: CgPythonTree) { element.value.accept(this) } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonVisitor.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonVisitor.kt index de93c4de42..d7540e30ec 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonVisitor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/constructor/visitor/CgPythonVisitor.kt @@ -14,7 +14,9 @@ interface CgPythonVisitor : CgVisitor { fun visit(element: CgPythonTuple): R fun visit(element: CgPythonList): R fun visit(element: CgPythonSet): R + fun visit(element: CgPythonIterator): R fun visit(element: CgPythonTree): R fun visit(element: CgPythonWith): R fun visit(element: CgPythonNamedArgument): R + fun visit(element: CgPythonZip): R } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/tree/CgPythonElement.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/tree/CgPythonElement.kt index 140eed8183..87269dd5d6 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/tree/CgPythonElement.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/codegen/model/tree/CgPythonElement.kt @@ -1,24 +1,22 @@ package org.utbot.python.framework.codegen.model.tree -import org.utbot.framework.codegen.domain.models.CgAnnotation -import org.utbot.framework.codegen.domain.models.CgDocumentationComment import org.utbot.framework.codegen.domain.models.CgElement import org.utbot.framework.codegen.domain.models.CgExpression import org.utbot.framework.codegen.domain.models.CgLiteral -import org.utbot.framework.codegen.domain.models.CgMethod -import org.utbot.framework.codegen.domain.models.CgMethodCall -import org.utbot.framework.codegen.domain.models.CgParameterDeclaration import org.utbot.framework.codegen.domain.models.CgStatement -import org.utbot.framework.codegen.domain.models.CgTestMethod -import org.utbot.framework.codegen.domain.models.CgTestMethodType import org.utbot.framework.codegen.domain.models.CgValue import org.utbot.framework.codegen.domain.models.CgVariable import org.utbot.framework.codegen.renderer.CgVisitor -import org.utbot.framework.codegen.tree.VisibilityModifier import org.utbot.framework.plugin.api.ClassId import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonTree -import org.utbot.python.framework.api.python.util.* +import org.utbot.python.framework.api.python.util.pythonDictClassId +import org.utbot.python.framework.api.python.util.pythonIntClassId +import org.utbot.python.framework.api.python.util.pythonIteratorClassId +import org.utbot.python.framework.api.python.util.pythonListClassId +import org.utbot.python.framework.api.python.util.pythonRangeClassId +import org.utbot.python.framework.api.python.util.pythonSetClassId +import org.utbot.python.framework.api.python.util.pythonTupleClassId import org.utbot.python.framework.codegen.model.constructor.visitor.CgPythonVisitor interface CgPythonElement : CgElement { @@ -37,6 +35,8 @@ interface CgPythonElement : CgElement { is CgPythonTree -> visitor.visit(element) is CgPythonWith -> visitor.visit(element) is CgPythonNamedArgument -> visitor.visit(element) + is CgPythonIterator -> visitor.visit(element) + is CgPythonZip -> visitor.visit(element) else -> throw IllegalArgumentException("Can not visit element of type ${element::class}") } } else { @@ -95,6 +95,14 @@ class CgPythonRange( ) } +class CgPythonZip( + val first: CgValue, + val second: CgValue +): CgValue, CgPythonElement { + override val type: ClassId + get() = PythonClassId("builtins.zip") +} + class CgPythonList( val elements: List ) : CgValue, CgPythonElement { @@ -119,6 +127,13 @@ class CgPythonDict( override val type: PythonClassId = pythonDictClassId } +class CgPythonIterator( + val elements: List, + val stopException: PythonClassId, +) : CgValue, CgPythonElement { + override val type: PythonClassId = pythonIteratorClassId +} + data class CgPythonWith( val expression: CgExpression, val target: CgExpression?, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/framework/external/PythonUtBotJavaApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/framework/external/PythonUtBotJavaApi.kt index 28d5e9bb55..879a0aac81 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/framework/external/PythonUtBotJavaApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/framework/external/PythonUtBotJavaApi.kt @@ -54,9 +54,9 @@ object PythonUtBotJavaApi { executionTimeout, ) logger.info("Loading information about Python types...") - val (mypyStorage, _) = processor.sourceCodeAnalyze() + val mypyConfig = processor.sourceCodeAnalyze() logger.info("Generating tests...") - return processor.testGenerate(mypyStorage) + return processor.testGenerate(mypyConfig) } /** diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt index 2d7f63ac35..3e86c2ef4e 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/PythonApi.kt @@ -1,56 +1,100 @@ package org.utbot.python.fuzzing import mu.KotlinLogging -import org.utbot.framework.plugin.api.UtError import org.utbot.fuzzer.FuzzedContext -import org.utbot.fuzzing.* +import org.utbot.fuzzing.Control +import org.utbot.fuzzing.Description +import org.utbot.fuzzing.Feedback +import org.utbot.fuzzing.Fuzzing +import org.utbot.fuzzing.Seed +import org.utbot.fuzzing.Statistic +import org.utbot.fuzzing.ValueProvider import org.utbot.fuzzing.utils.Trie import org.utbot.python.coverage.PyInstruction +import org.utbot.python.engine.ExecutionFeedback import org.utbot.python.framework.api.python.PythonTree -import org.utbot.python.framework.api.python.PythonUtExecution -import org.utbot.python.fuzzing.provider.* +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed +import org.utbot.python.fuzzing.provider.BoolValueProvider +import org.utbot.python.fuzzing.provider.BytearrayValueProvider +import org.utbot.python.fuzzing.provider.BytesValueProvider +import org.utbot.python.fuzzing.provider.ComplexValueProvider +import org.utbot.python.fuzzing.provider.ConstantValueProvider +import org.utbot.python.fuzzing.provider.DictValueProvider +import org.utbot.python.fuzzing.provider.FloatValueProvider +import org.utbot.python.fuzzing.provider.IntValueProvider +import org.utbot.python.fuzzing.provider.IteratorValueProvider +import org.utbot.python.fuzzing.provider.ListValueProvider +import org.utbot.python.fuzzing.provider.NoneValueProvider +import org.utbot.python.fuzzing.provider.OptionalValueProvider +import org.utbot.python.fuzzing.provider.RePatternValueProvider +import org.utbot.python.fuzzing.provider.ReduceValueProvider +import org.utbot.python.fuzzing.provider.SetValueProvider +import org.utbot.python.fuzzing.provider.StrValueProvider +import org.utbot.python.fuzzing.provider.SubtypeValueProvider +import org.utbot.python.fuzzing.provider.TupleFixSizeValueProvider +import org.utbot.python.fuzzing.provider.TupleValueProvider +import org.utbot.python.fuzzing.provider.TypeAliasValueProvider +import org.utbot.python.fuzzing.provider.UnionValueProvider import org.utbot.python.fuzzing.provider.utils.isAny -import org.utbot.python.newtyping.* +import org.utbot.python.newtyping.PythonTypeHintsStorage import org.utbot.python.newtyping.general.FunctionType import org.utbot.python.newtyping.general.UtType import org.utbot.python.newtyping.inference.InferredTypeFeedback import org.utbot.python.newtyping.inference.InvalidTypeFeedback import org.utbot.python.newtyping.inference.SuccessFeedback import org.utbot.python.newtyping.inference.baseline.BaselineAlgorithm +import org.utbot.python.newtyping.pythonModuleName +import org.utbot.python.newtyping.pythonName +import org.utbot.python.newtyping.pythonTypeName +import org.utbot.python.newtyping.pythonTypeRepresentation import org.utbot.python.utils.ExecutionWithTimoutMode +import org.utbot.python.utils.FakeWithTimeoutMode import org.utbot.python.utils.TestGenerationLimitManager -import org.utbot.python.utils.TimeoutMode import kotlin.random.Random private val logger = KotlinLogging.logger {} +typealias PythonValueProvider = ValueProvider + data class PythonFuzzedConcreteValue( val type: UtType, val value: Any, val fuzzedContext: FuzzedContext = FuzzedContext.Unknown, ) +data class FuzzedUtType( + val utType: UtType, + val fuzzAny: Boolean = false, +) { + + fun isAny(): Boolean = utType.isAny() + fun pythonName(): String = utType.pythonName() + fun pythonTypeName(): String = utType.pythonTypeName() + fun pythonModuleName(): String = utType.pythonModuleName() + fun pythonTypeRepresentation(): String = utType.pythonTypeRepresentation() + + companion object { + fun FuzzedUtType.activateAny() = FuzzedUtType(this.utType, true) + fun FuzzedUtType.activateAnyIf(parent: FuzzedUtType) = FuzzedUtType(this.utType, parent.fuzzAny) + fun Collection.activateAny() = this.map { it.activateAny() } + fun Collection.activateAnyIf(parent: FuzzedUtType) = this.map { it.activateAnyIf(parent) } + fun UtType.toFuzzed() = FuzzedUtType(this) + fun Collection.toFuzzed() = this.map { it.toFuzzed() } + } +} + class PythonMethodDescription( val name: String, - parameters: List, val concreteValues: Collection = emptyList(), val pythonTypeStorage: PythonTypeHintsStorage, val tracer: Trie, val random: Random, val limitManager: TestGenerationLimitManager, val type: FunctionType, -) : Description(parameters) - -sealed interface FuzzingExecutionFeedback -class ValidExecution(val utFuzzedExecution: PythonUtExecution): FuzzingExecutionFeedback -class InvalidExecution(val utError: UtError): FuzzingExecutionFeedback -class TypeErrorFeedback(val message: String) : FuzzingExecutionFeedback -class ArgumentsTypeErrorFeedback(val message: String) : FuzzingExecutionFeedback -class CachedExecutionFeedback(val cachedFeedback: FuzzingExecutionFeedback) : FuzzingExecutionFeedback -object FakeNodeFeedback : FuzzingExecutionFeedback +) : Description(type.arguments.toFuzzed()) data class PythonExecutionResult( - val fuzzingExecutionFeedback: FuzzingExecutionFeedback, + val executionFeedback: ExecutionFeedback, val fuzzingPlatformFeedback: PythonFeedback ) @@ -59,7 +103,7 @@ data class PythonFeedback( val result: Trie.Node = Trie.emptyNode(), val typeInferenceFeedback: InferredTypeFeedback = InvalidTypeFeedback, val fromCache: Boolean = false, -) : Feedback { +) : Feedback { fun fromCache(): PythonFeedback { return PythonFeedback( control = control, @@ -87,6 +131,7 @@ fun pythonDefaultValueProviders(typeStorage: PythonTypeHintsStorage) = listOf( DictValueProvider, TupleValueProvider, TupleFixSizeValueProvider, + OptionalValueProvider, UnionValueProvider, BytesValueProvider, BytearrayValueProvider, @@ -94,6 +139,7 @@ fun pythonDefaultValueProviders(typeStorage: PythonTypeHintsStorage) = listOf( RePatternValueProvider, ConstantValueProvider, TypeAliasValueProvider, + IteratorValueProvider, SubtypeValueProvider(typeStorage) ) @@ -112,11 +158,12 @@ fun pythonAnyTypeValueProviders() = listOf( class PythonFuzzing( private val pythonTypeStorage: PythonTypeHintsStorage, private val typeInferenceAlgorithm: BaselineAlgorithm, + private val globalIsCancelled: () -> Boolean, val execute: suspend (description: PythonMethodDescription, values: List) -> PythonFeedback, -) : Fuzzing { +) : Fuzzing { - private fun generateDefault(description: PythonMethodDescription, type: UtType)= sequence { - pythonDefaultValueProviders(pythonTypeStorage).asSequence().forEach { provider -> + private fun generateDefault(providers: List, description: PythonMethodDescription, type: FuzzedUtType)= sequence { + providers.asSequence().forEach { provider -> if (provider.accept(type)) { logger.debug { "Provider ${provider.javaClass.simpleName} accepts type ${type.pythonTypeRepresentation()}" } yieldAll(provider.generate(description, type)) @@ -124,16 +171,27 @@ class PythonFuzzing( } } - override fun generate(description: PythonMethodDescription, type: UtType): Sequence> { - var providers = emptyList>().asSequence() + private fun generateAnyProviders(description: PythonMethodDescription, type: FuzzedUtType) = sequence { + pythonAnyTypeValueProviders().asSequence().forEach { provider -> + logger.debug { "Provider ${provider.javaClass.simpleName} accepts type ${type.pythonTypeRepresentation()} with activated any" } + yieldAll(provider.generate(description, type)) + } + } + + override fun generate(description: PythonMethodDescription, type: FuzzedUtType): Sequence> { + val providers = mutableSetOf>() if (type.isAny()) { - logger.debug("Any does not have provider") + if (type.fuzzAny) { + providers += generateAnyProviders(description, type) + } else { + logger.debug("Any does not have provider") + } } else { - providers += generateDefault(description, type) + providers += generateDefault(pythonDefaultValueProviders(pythonTypeStorage), description, type) } - return providers + return providers.asSequence() } override suspend fun handle(description: PythonMethodDescription, values: List): PythonFeedback { @@ -147,19 +205,17 @@ class PythonFuzzing( return result } - private suspend fun forkType(description: PythonMethodDescription, stats: Statistic) { + private suspend fun forkType(description: PythonMethodDescription, stats: Statistic) { val type: UtType? = typeInferenceAlgorithm.expandState() if (type != null) { - val newTypes = (type as FunctionType).arguments val d = PythonMethodDescription( description.name, - newTypes, description.concreteValues, description.pythonTypeStorage, description.tracer, description.random, TestGenerationLimitManager(ExecutionWithTimoutMode, description.limitManager.until), - type + type as FunctionType ) if (!d.limitManager.isCancelled()) { logger.debug { "Fork new type" } @@ -167,18 +223,21 @@ class PythonFuzzing( } logger.debug { "Fork ended" } } else { - description.limitManager.mode = TimeoutMode + description.limitManager.mode = FakeWithTimeoutMode } } override suspend fun isCancelled( description: PythonMethodDescription, - stats: Statistic + stats: Statistic ): Boolean { + if (globalIsCancelled()) { + return true + } if (description.limitManager.isCancelled() || description.parameters.any { it.isAny() }) { forkType(description, stats) if (description.limitManager.isRootManager) { - return TimeoutMode.isCancelled(description.limitManager) + return FakeWithTimeoutMode.isCancelled(description.limitManager) } } return description.limitManager.isCancelled() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt index 17dbf7c2ef..945272d609 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BoolValueProvider.kt @@ -1,28 +1,27 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.fuzzing.seeds.Bool import org.utbot.fuzzing.seeds.KnownValue import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonBoolClassId +import org.utbot.python.fuzzing.FuzzedUtType import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.fuzzing.provider.utils.generateSummary -import org.utbot.python.newtyping.general.UtType -import org.utbot.python.newtyping.pythonTypeName -object BoolValueProvider : ValueProvider{ - override fun accept(type: UtType): Boolean { +object BoolValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonBoolClassId.canonicalName } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { yieldBool(Bool.TRUE()) { true } yieldBool(Bool.FALSE()) { false } } - private suspend fun > SequenceScope>.yieldBool(value: T, block: T.() -> Boolean) { + private suspend fun > SequenceScope>.yieldBool(value: T, block: T.() -> Boolean) { yield(Seed.Known(value) { PythonFuzzedValue( PythonTree.fromBool(block(it)), diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytearrayValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytearrayValueProvider.kt index 96f093c465..9c6b2765c3 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytearrayValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytearrayValueProvider.kt @@ -2,26 +2,25 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonBytearrayClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.newtyping.general.UtType -import org.utbot.python.newtyping.pythonTypeName -import org.utbot.python.newtyping.pythonTypeRepresentation +import org.utbot.python.fuzzing.PythonValueProvider -object BytearrayValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { +object BytearrayValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonBytearrayClassId.canonicalName } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { yield(Seed.Recursive( construct = Routine.Create( listOf( description.pythonTypeStorage.pythonInt, - ) + ).toFuzzed() ) { v -> val value = v.first().tree as PythonTree.PrimitiveNode PythonFuzzedValue( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytesValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytesValueProvider.kt index 9516a5c5a6..ba107fe158 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytesValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/BytesValueProvider.kt @@ -2,26 +2,25 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonBytesClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.newtyping.general.UtType -import org.utbot.python.newtyping.pythonTypeName -import org.utbot.python.newtyping.pythonTypeRepresentation +import org.utbot.python.fuzzing.PythonValueProvider -object BytesValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { +object BytesValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonBytesClassId.canonicalName } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { yield(Seed.Recursive( construct = Routine.Create( listOf( description.pythonTypeStorage.pythonInt, - ) + ).toFuzzed() ) { v -> val value = v.first().tree as PythonTree.PrimitiveNode PythonFuzzedValue( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt index c756cd20eb..3f8a65e0af 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ComplexValueProvider.kt @@ -2,55 +2,58 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonComplexClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.newtyping.createPythonUnionType -import org.utbot.python.newtyping.general.UtType -import org.utbot.python.newtyping.pythonTypeName -import org.utbot.python.newtyping.pythonTypeRepresentation -object ComplexValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { +object ComplexValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonComplexClassId.canonicalName } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { val numberType = createPythonUnionType( listOf( description.pythonTypeStorage.pythonFloat, description.pythonTypeStorage.pythonInt ) ) + val emptyValue = + PythonFuzzedValue( + PythonTree.PrimitiveNode( + pythonComplexClassId, + "complex()" + ), + "%var% = ${type.pythonTypeRepresentation()}" + ) yield(Seed.Recursive( construct = Routine.Create( listOf( numberType, numberType - ) + ).toFuzzed() ) { v -> - val real = v[0].tree as PythonTree.PrimitiveNode - val imag = v[1].tree as PythonTree.PrimitiveNode - val repr = "complex(real=${real.repr}, imag=${imag.repr})" - PythonFuzzedValue( - PythonTree.PrimitiveNode( - pythonComplexClassId, - repr - ), - "%var% = $repr" - ) + if (v[0].tree is PythonTree.FakeNode || v[1].tree is PythonTree.FakeNode) { + emptyValue + } else { + val real = v[0].tree as PythonTree.PrimitiveNode + val imag = v[1].tree as PythonTree.PrimitiveNode + val repr = "complex(real=${real.repr}, imag=${imag.repr})" + PythonFuzzedValue( + PythonTree.PrimitiveNode( + pythonComplexClassId, + repr + ), + "%var% = $repr" + ) + } }, - empty = Routine.Empty { - PythonFuzzedValue( - PythonTree.PrimitiveNode( - pythonComplexClassId, - "complex()" - ), - "%var% = ${type.pythonTypeRepresentation()}" - ) - } + empty = Routine.Empty { emptyValue } )) } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ConstantValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ConstantValueProvider.kt index 794c9c02d5..9201a17d94 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ConstantValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ConstantValueProvider.kt @@ -1,21 +1,20 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.fuzzing.FuzzedUtType import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.newtyping.general.UtType -import org.utbot.python.newtyping.pythonTypeName +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.fuzzing.value.TypesFromJSONStorage -object ConstantValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { +object ConstantValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return TypesFromJSONStorage.getTypesFromJsonStorage().containsKey(type.pythonTypeName()) } - override fun generate(description: PythonMethodDescription, type: UtType): Sequence> = + override fun generate(description: PythonMethodDescription, type: FuzzedUtType): Sequence> = sequence { val storage = TypesFromJSONStorage.getTypesFromJsonStorage() storage.values.forEach { values -> diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt index 22accbbf78..a005fb7862 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/DictValueProvider.kt @@ -2,54 +2,41 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonDictClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAnyIf +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.newtyping.general.UtType +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.newtyping.pythonAnnotationParameters -import org.utbot.python.newtyping.pythonTypeName -import org.utbot.python.newtyping.pythonTypeRepresentation -object DictValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { +object DictValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonDictClassId.canonicalName } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { - val params = type.pythonAnnotationParameters() + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { + val params = type.utType.pythonAnnotationParameters() - val modifications = emptyList>().toMutableList() - modifications.add(Routine.Call(params) { instance, arguments -> - val key = arguments[0].tree - val value = arguments[1].tree - val dict = instance.tree as PythonTree.DictNode - if (dict.items.keys.toList().contains(key)) { - dict.items.replace(key, value) - } else { - dict.items[key] = value - } - }) - modifications.add(Routine.Call(listOf(params[0])) { instance, arguments -> - val key = arguments[0].tree - val dict = instance.tree as PythonTree.DictNode - if (dict.items.keys.toList().contains(key)) { - dict.items.remove(key) - } - }) - yield(Seed.Recursive( - construct = Routine.Create(emptyList()) { v -> + yield(Seed.Collection( + construct = Routine.Collection { _ -> PythonFuzzedValue( PythonTree.DictNode(mutableMapOf()), "%var% = ${type.pythonTypeRepresentation()}" ) }, - modify = modifications.asSequence(), - empty = Routine.Empty { PythonFuzzedValue( - PythonTree.DictNode(emptyMap().toMutableMap()), - "%var% = ${type.pythonTypeRepresentation()}" - )} + modify = Routine.ForEach(params.toFuzzed().activateAnyIf(type)) { instance, _, arguments -> + val key = arguments[0].tree + val value = arguments[1].tree + val dict = instance.tree as PythonTree.DictNode + if (dict.items.keys.toList().contains(key)) { + dict.items.replace(key, value) + } else { + dict.items[key] = value + } + }, )) } -} \ No newline at end of file +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt index 496ac4b67a..d6089aa2fa 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/FloatValueProvider.kt @@ -1,28 +1,29 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.fuzzing.seeds.IEEE754Value import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonFloatClassId import org.utbot.python.framework.api.python.util.pythonIntClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedConcreteValue import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.fuzzing.provider.utils.generateSummary -import org.utbot.python.newtyping.general.UtType import org.utbot.python.newtyping.pythonTypeName import java.math.BigDecimal import java.math.BigInteger -object FloatValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { +object FloatValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonFloatClassId.canonicalName } private fun getFloatConstants(concreteValues: Collection): List { return concreteValues - .filter { accept(it.type) } + .filter { accept(it.type.toFuzzed()) } .map { fuzzedValue -> (fuzzedValue.value as BigDecimal).let { IEEE754Value.fromValue(it.toDouble()) @@ -40,7 +41,7 @@ object FloatValueProvider : ValueProvider> = sequence { + override fun generate(description: PythonMethodDescription, type: FuzzedUtType): Sequence> = sequence { val floatConstants = getFloatConstants(description.concreteValues) val intConstants = getIntConstants(description.concreteValues) val constants = floatConstants + intConstants + listOf(0, 1).map { IEEE754Value.fromValue(it.toDouble()) } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt index cf4f6d36dc..3f4dd7e989 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IntValueProvider.kt @@ -3,26 +3,26 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Configuration import org.utbot.fuzzing.Mutation import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.fuzzing.seeds.BitVectorValue import org.utbot.fuzzing.seeds.KnownValue import org.utbot.fuzzing.seeds.Signed import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonIntClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedConcreteValue import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.fuzzing.provider.utils.generateSummary -import org.utbot.python.newtyping.general.UtType -import org.utbot.python.newtyping.pythonTypeName import java.math.BigInteger import kotlin.random.Random -object IntValueProvider : ValueProvider { +object IntValueProvider : PythonValueProvider { private val randomStubWithNoUsage = Random(0) private val configurationStubWithNoUsage = Configuration() - override fun accept(type: UtType): Boolean { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonIntClassId.canonicalName } @@ -34,7 +34,7 @@ object IntValueProvider : ValueProvider): List { return concreteValues - .filter { accept(it.type) } + .filter { accept(it.type.toFuzzed()) } .map { fuzzedValue -> (fuzzedValue.value as BigInteger).let { BitVectorValue.fromBigInteger(it) @@ -42,7 +42,7 @@ object IntValueProvider : ValueProvider> { + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence> { val bits = 128 val integerConstants = getIntConstants(description.concreteValues) val modifiedConstants = integerConstants.flatMap { value -> diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IteratorValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IteratorValueProvider.kt new file mode 100644 index 0000000000..026e9e1d10 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/IteratorValueProvider.kt @@ -0,0 +1,37 @@ +package org.utbot.python.fuzzing.provider + +import org.utbot.fuzzing.Routine +import org.utbot.fuzzing.Seed +import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.framework.api.python.util.pythonIteratorClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAnyIf +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed +import org.utbot.python.fuzzing.PythonFuzzedValue +import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider +import org.utbot.python.newtyping.pythonAnnotationParameters + +object IteratorValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { + return type.pythonTypeName() == pythonIteratorClassId.canonicalName + } + + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { + val param = type.utType.pythonAnnotationParameters() + yield( + Seed.Collection( + construct = Routine.Collection { + PythonFuzzedValue( + PythonTree.IteratorNode( + emptyMap().toMutableMap(), + ), + "%var% = ${type.pythonTypeRepresentation()}" + ) + }, + modify = Routine.ForEach(param.toFuzzed().activateAnyIf(type)) { self, i, values -> + (self.tree as PythonTree.IteratorNode).items[i] = values.first().tree + } + )) + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ListValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ListValueProvider.kt index 94c23cf191..ae19ff30f1 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ListValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ListValueProvider.kt @@ -2,23 +2,23 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonListClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAnyIf +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.newtyping.general.UtType +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.newtyping.pythonAnnotationParameters -import org.utbot.python.newtyping.pythonTypeName -import org.utbot.python.newtyping.pythonTypeRepresentation -object ListValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { +object ListValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonListClassId.canonicalName } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { - val param = type.pythonAnnotationParameters() + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { + val param = type.utType.pythonAnnotationParameters() yield( Seed.Collection( construct = Routine.Collection { @@ -29,7 +29,7 @@ object ListValueProvider : ValueProvider + modify = Routine.ForEach(param.toFuzzed().activateAnyIf(type)) { self, i, values -> (self.tree as PythonTree.ListNode).items[i] = values.first().tree } )) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/NoneValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/NoneValueProvider.kt index 9d16d2db3f..3ea60ccfb4 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/NoneValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/NoneValueProvider.kt @@ -1,19 +1,19 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.fuzzing.FuzzedUtType import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.newtyping.PythonNoneTypeDescription -import org.utbot.python.newtyping.general.UtType -object NoneValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { - return type.meta is PythonNoneTypeDescription +object NoneValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { + return type.utType.meta is PythonNoneTypeDescription } - override fun generate(description: PythonMethodDescription, type: UtType): Sequence> = sequence { + override fun generate(description: PythonMethodDescription, type: FuzzedUtType): Sequence> = sequence { yield(Seed.Simple(PythonFuzzedValue(PythonTree.fromNone(), "%var% = None"))) } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/OptionalValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/OptionalValueProvider.kt new file mode 100644 index 0000000000..121bf97b00 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/OptionalValueProvider.kt @@ -0,0 +1,30 @@ +package org.utbot.python.fuzzing.provider + +import org.utbot.fuzzing.Routine +import org.utbot.fuzzing.Seed +import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAnyIf +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed +import org.utbot.python.fuzzing.PythonFuzzedValue +import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider +import org.utbot.python.newtyping.PythonNoneTypeDescription +import org.utbot.python.newtyping.PythonUnionTypeDescription +import org.utbot.python.newtyping.pythonAnnotationParameters + +object OptionalValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { + return type.utType.meta is PythonUnionTypeDescription && type.utType.parameters.any { it.meta is PythonNoneTypeDescription } + } + + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { + val params = type.utType.pythonAnnotationParameters() + params.forEach { unionParam -> + yield(Seed.Recursive( + construct = Routine.Create(listOf(unionParam).toFuzzed().activateAnyIf(type)) { v -> v.first() }, + empty = Routine.Empty { PythonFuzzedValue(PythonTree.fromNone()) } + )) + } + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/RePatternValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/RePatternValueProvider.kt index 4a47f7575d..aaf2075391 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/RePatternValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/RePatternValueProvider.kt @@ -2,34 +2,28 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider -import org.utbot.fuzzing.seeds.Bool -import org.utbot.fuzzing.seeds.KnownValue import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonTree -import org.utbot.python.framework.api.python.util.pythonBoolClassId import org.utbot.python.framework.api.python.util.pythonRePatternClassId import org.utbot.python.framework.api.python.util.toPythonRepr +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.fuzzing.provider.utils.generateSummary -import org.utbot.python.fuzzing.provider.utils.isAny +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.fuzzing.provider.utils.makeRawString -import org.utbot.python.fuzzing.provider.utils.transformRawString -import org.utbot.python.newtyping.general.UtType -import org.utbot.python.newtyping.pythonTypeName -object RePatternValueProvider : ValueProvider{ - override fun accept(type: UtType): Boolean { +object RePatternValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonRePatternClassId.canonicalName } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { yield(Seed.Recursive( construct = Routine.Create( listOf( description.pythonTypeStorage.pythonStr, - ) + ).toFuzzed() ) { v -> val value = v.first().tree as PythonTree.PrimitiveNode val rawValue = value.repr.toPythonRepr().makeRawString() diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt index 5bcf23358a..532ebd405d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/ReduceValueProvider.kt @@ -2,12 +2,28 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonClassId import org.utbot.python.framework.api.python.PythonTree -import org.utbot.python.framework.api.python.util.* +import org.utbot.python.framework.api.python.util.pythonBoolClassId +import org.utbot.python.framework.api.python.util.pythonBytearrayClassId +import org.utbot.python.framework.api.python.util.pythonBytesClassId +import org.utbot.python.framework.api.python.util.pythonComplexClassId +import org.utbot.python.framework.api.python.util.pythonDictClassId +import org.utbot.python.framework.api.python.util.pythonFloatClassId +import org.utbot.python.framework.api.python.util.pythonIntClassId +import org.utbot.python.framework.api.python.util.pythonListClassId +import org.utbot.python.framework.api.python.util.pythonObjectClassId +import org.utbot.python.framework.api.python.util.pythonRePatternClassId +import org.utbot.python.framework.api.python.util.pythonSetClassId +import org.utbot.python.framework.api.python.util.pythonStrClassId +import org.utbot.python.framework.api.python.util.pythonTupleClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAny +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAnyIf +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.fuzzing.provider.utils.isAny import org.utbot.python.fuzzing.provider.utils.isCallable import org.utbot.python.fuzzing.provider.utils.isConcreteType @@ -15,11 +31,17 @@ import org.utbot.python.fuzzing.provider.utils.isMagic import org.utbot.python.fuzzing.provider.utils.isPrivate import org.utbot.python.fuzzing.provider.utils.isProperty import org.utbot.python.fuzzing.provider.utils.isProtected -import org.utbot.python.newtyping.* +import org.utbot.python.newtyping.PythonCallableTypeDescription +import org.utbot.python.newtyping.PythonCompositeTypeDescription +import org.utbot.python.newtyping.PythonDefinition import org.utbot.python.newtyping.general.FunctionType -import org.utbot.python.newtyping.general.UtType +import org.utbot.python.newtyping.getPythonAttributeByName +import org.utbot.python.newtyping.getPythonAttributes +import org.utbot.python.newtyping.pythonDescription +import org.utbot.python.newtyping.pythonNoneType +import org.utbot.python.newtyping.pythonTypeName -object ReduceValueProvider : ValueProvider { +object ReduceValueProvider : PythonValueProvider { private val unsupportedTypes = listOf( pythonRePatternClassId.canonicalName, pythonListClassId.canonicalName, @@ -33,74 +55,174 @@ object ReduceValueProvider : ValueProvider>().toMutableList() + modifications.addAll(fields.map { field -> + Routine.Call(listOf(field.type).toFuzzed().activateAny()) { instance, arguments -> + val obj = instance.tree as PythonTree.ReduceNode + obj.state[field.meta.name] = arguments.first().tree + } + }) + + val (constructors, newType) = findConstructors(description, type) + constructors .forEach { - val modifications = emptyList>().toMutableList() - modifications.addAll(fields.map { field -> - Routine.Call(listOf(field.type)) { instance, arguments -> - val obj = instance.tree as PythonTree.ReduceNode - obj.state[field.meta.name] = arguments.first().tree - } - }) - yieldAll(callConstructors(type, it, modifications.asSequence())) + yieldAll(callConstructors(newType, it, modifications.asSequence(), description)) } } - private fun findFields(description: PythonMethodDescription, type: UtType): List { + private fun findFields(description: PythonMethodDescription, type: FuzzedUtType): List { // TODO: here we need to use same as .getPythonAttributeByName but without name // TODO: now we do not have fields from parents // TODO: here we should use only attributes from __slots__ - return type.getPythonAttributes().filter { attr -> + return type.utType.getPythonAttributes().filter { attr -> !attr.isMagic() && !attr.isProtected() && !attr.isPrivate() && !attr.isProperty() && !attr.isCallable( description.pythonTypeStorage ) } } - private fun findConstructors(description: PythonMethodDescription, type: UtType): List { + /* + * 1. Annotated __init__ without functional arguments and mro(__init__) <= mro(__new__) -> listOf(__init__) + * 2. Not 1 and annotated __new__ without functional arguments -> listOf(__new__) + * 3. Not 1 and not 2 and __init__ without tp.Any -> listOf(__init__) + * 4. Not 1 and not 2 and not 3 and type is not generic -> listOf(__init__, __new__) + activateAny + * 5. emptyList() + */ + private fun findConstructors(description: PythonMethodDescription, type: FuzzedUtType): Pair, FuzzedUtType> { val initMethodName = "__init__" val newMethodName = "__new__" - val typeDescr = type.pythonDescription() + val typeDescr = type.utType.pythonDescription() return if (typeDescr is PythonCompositeTypeDescription) { - val mro = typeDescr.mro(description.pythonTypeStorage, type) - val initParent = mro.indexOfFirst { p -> p.getPythonAttributes().any { it.meta.name == initMethodName } } - val newParent = mro.indexOfFirst { p -> p.getPythonAttributes().any { it.meta.name == newMethodName } } - val initMethods = type.getPythonAttributeByName(description.pythonTypeStorage, initMethodName) - val newMethods = type.getPythonAttributeByName(description.pythonTypeStorage, newMethodName) - if (initParent <= newParent && initMethods != null) { - listOf(initMethods) - } else if (newMethods != null) { - listOf(newMethods) - } else { - emptyList() // probably not reachable (because of class object) - } + val mro = typeDescr.mro(description.pythonTypeStorage, type.utType) + val initParent = mro.indexOfFirst { p -> p.getPythonAttributes().any { it.meta.name == initMethodName } } + val newParent = mro.indexOfFirst { p -> p.getPythonAttributes().any { it.meta.name == newMethodName } } + val initMethod = type.utType.getPythonAttributeByName(description.pythonTypeStorage, initMethodName) + val newMethod = type.utType.getPythonAttributeByName(description.pythonTypeStorage, newMethodName) + + val initWithoutCallable = initMethod?.isCallable(description.pythonTypeStorage) ?: false + val newWithoutCallable = newMethod?.isCallable(description.pythonTypeStorage) ?: false + + val initWithoutAny = (initMethod?.type as? FunctionType)?.arguments?.drop(1)?.all { !it.isAny() } ?: false + val newWithoutAny = (initMethod?.type as? FunctionType)?.arguments?.drop(1)?.all { !it.isAny() } ?: false + + if (initParent <= newParent && initMethod != null && initWithoutCallable && initWithoutAny) { + listOf(initMethod) to type + } else if (newMethod != null && newWithoutCallable && newWithoutAny) { + listOf(newMethod) to type + } else if (initMethod != null && initWithoutAny) { + listOf(initMethod) to type } else { - emptyList() + listOfNotNull(initMethod, newMethod) to type.activateAny() } + } else { + emptyList() to type + } } - private fun constructObject( - type: UtType, - constructorFunction: FunctionType, - modifications: Sequence> - ): Seed.Recursive { - val description = constructorFunction.pythonDescription() as PythonCallableTypeDescription - val positionalArgs = description.argumentKinds.count { it == PythonCallableTypeDescription.ArgKind.ARG_POS } - val arguments = constructorFunction.arguments.take(positionalArgs) - val nonSelfArgs = arguments.drop(1) + private fun callConstructors( + type: FuzzedUtType, + constructor: PythonDefinition, + modifications: Sequence>, + description: PythonMethodDescription, + ): Sequence> = sequence { + val constructors = emptyList>().toMutableList() + if (constructor.type.pythonTypeName() == "Overload") { + constructor.type.parameters.forEach { + if (it is FunctionType) { + constructors.add(it to constructor.meta.name) + } + } + } else { + constructors.add(constructor.type as FunctionType to constructor.meta.name) + } + constructors.forEach { + yield(constructObject(type, it, modifications, description)) + } + } + private fun constructObject( + type: FuzzedUtType, + constructorFunction: Pair, + modifications: Sequence>, + description: PythonMethodDescription, + ): Seed.Recursive { return Seed.Recursive( - construct = Routine.Create(nonSelfArgs) { v -> + construct = buildConstructor(type, constructorFunction, description), + modify = modifications, + empty = Routine.Empty { PythonFuzzedValue(buildEmptyValue(type, description)) } + ) + } + + private fun buildEmptyValue( + type: FuzzedUtType, + description: PythonMethodDescription, + ): PythonTree.PythonTreeNode { + val newMethodName = "__new__" + val newMethod = type.utType.getPythonAttributeByName(description.pythonTypeStorage, newMethodName) + return if (newMethod?.type?.parameters?.size == 1) { + val classId = PythonClassId(type.pythonModuleName(), type.pythonName()) + PythonTree.ReduceNode( + classId, + PythonClassId(type.pythonModuleName(), "${type.pythonName()}.__new__"), + listOf(PythonTree.PrimitiveNode(classId, classId.name)), + ) + } else { + PythonTree.FakeNode + } + } + + private fun buildConstructor( + type: FuzzedUtType, + constructor: Pair, + description: PythonMethodDescription, + ): Routine.Create { + val (constructorFunction, constructorName) = constructor + val newMethodName = "__new__" + if (constructorName == newMethodName) { + val newMethodArgs = constructorFunction.arguments + return if (newMethodArgs.size == 1) { + val classId = PythonClassId(type.pythonModuleName(), type.pythonName()) + Routine.Create(listOf(pythonNoneType).toFuzzed()) { _ -> + PythonFuzzedValue( + PythonTree.ReduceNode( + classId, + PythonClassId(type.pythonModuleName(), "${type.pythonName()}.__new__"), + listOf(PythonTree.PrimitiveNode(classId, classId.name)), + ), + "%var% = ${type.pythonTypeRepresentation()}" + ) + } + } else { + // TODO: remove it + val classId = PythonClassId(type.pythonModuleName(), type.pythonName()) + Routine.Create(listOf(pythonNoneType).toFuzzed()) { v -> + PythonFuzzedValue( + PythonTree.ReduceNode( + classId, + PythonClassId(pythonObjectClassId.name, "${pythonObjectClassId.name}.__new__"), + listOf(PythonTree.PrimitiveNode(classId, classId.name)), + ), + "%var% = ${type.pythonTypeRepresentation()}" + ) + } + } + } else { + val typeDescription = constructorFunction.pythonDescription() as PythonCallableTypeDescription + val positionalArgs = typeDescription.argumentKinds.count { it == PythonCallableTypeDescription.ArgKind.ARG_POS } + val arguments = constructorFunction.arguments.take(positionalArgs) + val nonSelfArgs = arguments.drop(1) + return Routine.Create(nonSelfArgs.toFuzzed().activateAnyIf(type)) { v -> PythonFuzzedValue( PythonTree.ReduceNode( PythonClassId(type.pythonModuleName(), type.pythonName()), @@ -109,29 +231,8 @@ object ReduceValueProvider : ValueProvider> - ): Sequence> = sequence { - val constructors = emptyList().toMutableList() - if (constructor.type.pythonTypeName() == "Overload") { - constructor.type.parameters.forEach { - if (it is FunctionType) { - constructors.add(it) - } } - } else { - constructors.add(constructor.type as FunctionType) - } - constructors.forEach { - yield(constructObject(type, it, modifications)) } } + } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SetValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SetValueProvider.kt index c6f7e6aef7..37d8ca0b92 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SetValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SetValueProvider.kt @@ -2,49 +2,36 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonSetClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAnyIf +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.newtyping.general.UtType +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.newtyping.pythonAnnotationParameters -import org.utbot.python.newtyping.pythonTypeName -import org.utbot.python.newtyping.pythonTypeRepresentation -object SetValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { +object SetValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonSetClassId.canonicalName } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { - val params = type.pythonAnnotationParameters() + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { + val params = type.utType.pythonAnnotationParameters() - val modifications = emptyList>().toMutableList() - modifications.add(Routine.Call(params) { instance, arguments -> - val set = instance.tree as PythonTree.SetNode - set.items.add(arguments.first().tree) - }) - modifications.add(Routine.Call(params) { instance, arguments -> - val set = instance.tree as PythonTree.SetNode - val value = arguments.first().tree - if (set.items.contains(value)) { - set.items.remove(value) - } - }) - yield(Seed.Recursive( - construct = Routine.Create(emptyList()) { - val items = emptySet().toMutableSet() + yield(Seed.Collection( + construct = Routine.Collection { _ -> PythonFuzzedValue( - PythonTree.SetNode(items), - "%var% = ${type.pythonTypeRepresentation()}", + PythonTree.SetNode(mutableSetOf()), + "%var% = ${type.pythonTypeRepresentation()}" ) }, - modify = modifications.asSequence(), - empty = Routine.Empty { PythonFuzzedValue( - PythonTree.SetNode(emptySet().toMutableSet()), - "%var% = ${type.pythonTypeRepresentation()}", - )} + modify = Routine.ForEach(params.toFuzzed().activateAnyIf(type)) { instance, _, arguments -> + val item = arguments[0].tree + val set = instance.tree as PythonTree.SetNode + set.items.add(item) + }, )) } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt index 2f035ca942..51cf2b62cd 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/StrValueProvider.kt @@ -1,30 +1,30 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.fuzzing.seeds.KnownValue import org.utbot.fuzzing.seeds.RegexValue import org.utbot.fuzzing.seeds.StringValue import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonStrClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedConcreteValue import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.fuzzing.provider.utils.generateSummary import org.utbot.python.fuzzing.provider.utils.isPattern import org.utbot.python.fuzzing.provider.utils.transformQuotationMarks import org.utbot.python.fuzzing.provider.utils.transformRawString -import org.utbot.python.newtyping.general.UtType -import org.utbot.python.newtyping.pythonTypeName -object StrValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { +object StrValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonStrClassId.canonicalName } private fun getConstants(concreteValues: Collection): List { return concreteValues - .filter { accept(it.type) } + .filter { accept(it.type.toFuzzed()) } .map { it.value as String } } @@ -40,7 +40,7 @@ object StrValueProvider : ValueProvider> SequenceScope>.yieldStrings(value: T, block: T.() -> Any) { + private suspend fun > SequenceScope>.yieldStrings(value: T, block: T.() -> Any) { yield(Seed.Known(value) { PythonFuzzedValue( PythonTree.fromString(block(it).toString()), diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SubtypeValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SubtypeValueProvider.kt index 5b7289f6c9..00d37b1e7e 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SubtypeValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/SubtypeValueProvider.kt @@ -2,22 +2,28 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAnyIf +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.fuzzing.provider.utils.isConcreteType -import org.utbot.python.newtyping.* +import org.utbot.python.newtyping.PythonConcreteCompositeTypeDescription +import org.utbot.python.newtyping.PythonProtocolDescription import org.utbot.python.newtyping.PythonSubtypeChecker.Companion.checkIfRightIsSubtypeOfLeft +import org.utbot.python.newtyping.PythonTypeHintsStorage import org.utbot.python.newtyping.general.DefaultSubstitutionProvider -import org.utbot.python.newtyping.general.UtType +import org.utbot.python.newtyping.pythonAnyType +import org.utbot.python.newtyping.pythonDescription class SubtypeValueProvider( private val typeStorage: PythonTypeHintsStorage -) : ValueProvider { - override fun accept(type: UtType): Boolean { - return type.meta is PythonProtocolDescription || - ((type.meta as? PythonConcreteCompositeTypeDescription)?.isAbstract == true) +) : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { + return type.utType.meta is PythonProtocolDescription || + ((type.utType.meta as? PythonConcreteCompositeTypeDescription)?.isAbstract == true) } private val concreteTypes = typeStorage.simpleTypes.filter { @@ -26,14 +32,14 @@ class SubtypeValueProvider( DefaultSubstitutionProvider.substituteAll(it, it.parameters.map { pythonAnyType }) } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { - val subtypes = concreteTypes.filter { checkIfRightIsSubtypeOfLeft(type, it, typeStorage) } + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { + val subtypes = concreteTypes.filter { checkIfRightIsSubtypeOfLeft(type.utType, it, typeStorage) } subtypes.forEach { subtype -> yield( Seed.Recursive( - construct = Routine.Create(listOf(subtype)) { v -> v.first() }, - empty = Routine.Empty { PythonFuzzedValue(PythonTree.FakeNode) } - )) + construct = Routine.Create(listOf(subtype).toFuzzed().activateAnyIf(type)) { v -> v.first() }, + empty = Routine.Empty { PythonFuzzedValue(PythonTree.FakeNode) } + )) } } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleFixSizeValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleFixSizeValueProvider.kt index 3e39163faa..e8de4c6f95 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleFixSizeValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleFixSizeValueProvider.kt @@ -2,31 +2,32 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAnyIf +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.newtyping.PythonTupleTypeDescription -import org.utbot.python.newtyping.general.UtType import org.utbot.python.newtyping.pythonAnnotationParameters -import org.utbot.python.newtyping.pythonTypeRepresentation -object TupleFixSizeValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { - return type.meta is PythonTupleTypeDescription +object TupleFixSizeValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { + return type.utType.meta is PythonTupleTypeDescription } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { - val params = type.pythonAnnotationParameters() + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { + val params = type.utType.pythonAnnotationParameters() val length = params.size - val modifications = emptyList>().toMutableList() + val modifications = emptyList>().toMutableList() for (i in 0 until length) { - modifications.add(Routine.Call(listOf(params[i])) { instance, arguments -> + modifications.add(Routine.Call(listOf(params[i]).toFuzzed().activateAnyIf(type)) { instance, arguments -> (instance.tree as PythonTree.TupleNode).items[i] = arguments.first().tree }) } yield(Seed.Recursive( - construct = Routine.Create(params) { v -> + construct = Routine.Create(params.toFuzzed().activateAnyIf(type)) { v -> PythonFuzzedValue( PythonTree.TupleNode(v.withIndex().associate { it.index to it.value.tree }.toMutableMap()), "%var% = ${type.pythonTypeRepresentation()}" diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleValueProvider.kt index a0ac24de31..e7de9bfa6b 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TupleValueProvider.kt @@ -2,23 +2,27 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.util.pythonTupleClassId +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAnyIf +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.fuzzing.provider.utils.getSuitableConstantsFromCode -import org.utbot.python.newtyping.* -import org.utbot.python.newtyping.general.UtType +import org.utbot.python.fuzzing.PythonValueProvider +import org.utbot.python.newtyping.PythonSubtypeChecker +import org.utbot.python.newtyping.pythonAnnotationParameters +import org.utbot.python.newtyping.pythonAnyType +import org.utbot.python.newtyping.typesAreEqual -object TupleValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { +object TupleValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { return type.pythonTypeName() == pythonTupleClassId.canonicalName } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { yieldAll(getConstants(description, type)) - val param = type.pythonAnnotationParameters() + val param = type.utType.pythonAnnotationParameters() yield( Seed.Collection( construct = Routine.Collection { @@ -29,15 +33,25 @@ object TupleValueProvider : ValueProvider + modify = Routine.ForEach(param.toFuzzed().activateAnyIf(type)) { self, i, values -> (self.tree as PythonTree.TupleNode).items[i] = values.first().tree } )) } - private fun getConstants(description: PythonMethodDescription, type: UtType): List> { - if (!typesAreEqual(type.parameters.first(), pythonAnyType)) + private fun getConstants(description: PythonMethodDescription, type: FuzzedUtType): List> { + if (!typesAreEqual(type.utType.parameters.first(), pythonAnyType)) return getSuitableConstantsFromCode(description, type) return emptyList() } + private fun getSuitableConstantsFromCode(description: PythonMethodDescription, type: FuzzedUtType): List> { + return description.concreteValues.filter { + PythonSubtypeChecker.checkIfRightIsSubtypeOfLeft(type.utType, it.type, description.pythonTypeStorage) + }.mapNotNull { value -> + PythonTree.fromParsedConstant(Pair(value.type, value.value))?.let { + Seed.Simple(PythonFuzzedValue(it)) + } + } + } + } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TypeAliasValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TypeAliasValueProvider.kt index ffc4a4621c..cf46ce04ca 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TypeAliasValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/TypeAliasValueProvider.kt @@ -2,24 +2,28 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAnyIf +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider import org.utbot.python.newtyping.PythonTypeAliasDescription -import org.utbot.python.newtyping.general.UtType -object TypeAliasValueProvider : ValueProvider { - - override fun accept(type: UtType): Boolean { - return type.meta is PythonTypeAliasDescription +object TypeAliasValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { + return type.utType.meta is PythonTypeAliasDescription } - override fun generate(description: PythonMethodDescription, type: UtType): Sequence> { - val compositeType = PythonTypeAliasDescription.castToCompatibleTypeApi(type) + override fun generate( + description: PythonMethodDescription, + type: FuzzedUtType + ): Sequence> { + val compositeType = PythonTypeAliasDescription.castToCompatibleTypeApi(type.utType) return sequenceOf( Seed.Recursive( - construct = Routine.Create(listOf(compositeType.members[0])) { v -> v.first() }, + construct = Routine.Create(listOf(compositeType.members[0]).toFuzzed().activateAnyIf(type)) { v -> v.first() }, empty = Routine.Empty { PythonFuzzedValue(PythonTree.FakeNode) } ) ) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/UnionValueProvider.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/UnionValueProvider.kt index 86388e67ea..7316f99deb 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/UnionValueProvider.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/UnionValueProvider.kt @@ -2,24 +2,27 @@ package org.utbot.python.fuzzing.provider import org.utbot.fuzzing.Routine import org.utbot.fuzzing.Seed -import org.utbot.fuzzing.ValueProvider import org.utbot.python.framework.api.python.PythonTree +import org.utbot.python.fuzzing.FuzzedUtType +import org.utbot.python.fuzzing.FuzzedUtType.Companion.activateAnyIf +import org.utbot.python.fuzzing.FuzzedUtType.Companion.toFuzzed import org.utbot.python.fuzzing.PythonFuzzedValue import org.utbot.python.fuzzing.PythonMethodDescription +import org.utbot.python.fuzzing.PythonValueProvider +import org.utbot.python.newtyping.PythonNoneTypeDescription import org.utbot.python.newtyping.PythonUnionTypeDescription -import org.utbot.python.newtyping.general.UtType import org.utbot.python.newtyping.pythonAnnotationParameters -object UnionValueProvider : ValueProvider { - override fun accept(type: UtType): Boolean { - return type.meta is PythonUnionTypeDescription +object UnionValueProvider : PythonValueProvider { + override fun accept(type: FuzzedUtType): Boolean { + return type.utType.meta is PythonUnionTypeDescription && type.utType.parameters.all { it.meta !is PythonNoneTypeDescription } } - override fun generate(description: PythonMethodDescription, type: UtType) = sequence { - val params = type.pythonAnnotationParameters() + override fun generate(description: PythonMethodDescription, type: FuzzedUtType) = sequence { + val params = type.utType.pythonAnnotationParameters() params.forEach { unionParam -> yield(Seed.Recursive( - construct = Routine.Create(listOf(unionParam)) { v -> v.first() }, + construct = Routine.Create(listOf(unionParam).toFuzzed().activateAnyIf(type)) { v -> v.first() }, empty = Routine.Empty { PythonFuzzedValue(PythonTree.FakeNode) } )) } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/utils/ProviderUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/utils/ProviderUtils.kt index 633956d31e..30204cdae4 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/utils/ProviderUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/fuzzing/provider/utils/ProviderUtils.kt @@ -1,26 +1,17 @@ package org.utbot.python.fuzzing.provider.utils -import org.utbot.fuzzing.Seed -import org.utbot.python.framework.api.python.PythonTree -import org.utbot.python.fuzzing.PythonFuzzedValue -import org.utbot.python.fuzzing.PythonMethodDescription -import org.utbot.python.newtyping.* +import org.utbot.python.newtyping.PythonAnyTypeDescription +import org.utbot.python.newtyping.PythonConcreteCompositeTypeDescription +import org.utbot.python.newtyping.PythonDefinition +import org.utbot.python.newtyping.PythonTypeHintsStorage +import org.utbot.python.newtyping.PythonVariableDescription import org.utbot.python.newtyping.general.UtType +import org.utbot.python.newtyping.getPythonAttributeByName fun UtType.isAny(): Boolean { return meta is PythonAnyTypeDescription } -fun getSuitableConstantsFromCode(description: PythonMethodDescription, type: UtType): List> { - return description.concreteValues.filter { - PythonSubtypeChecker.checkIfRightIsSubtypeOfLeft(type, it.type, description.pythonTypeStorage) - }.mapNotNull { value -> - PythonTree.fromParsedConstant(Pair(value.type, value.value))?.let { - Seed.Simple(PythonFuzzedValue(it)) - } - } -} - fun isConcreteType(type: UtType): Boolean { return (type.meta as? PythonConcreteCompositeTypeDescription)?.isAbstract == false } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/ParseUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/ParseUtils.kt index 67fe571639..eb1fb54d15 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/ParseUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/ParseUtils.kt @@ -4,7 +4,7 @@ import org.parsers.python.Node import org.parsers.python.PythonConstants import org.parsers.python.ast.* -data class ParsedFunctionDefinition(val name: Name, val body: Block) +data class ParsedFunctionDefinition(val name: Name, val body: Block, val decorators: List) data class ParsedClassDefinition(val name: Name, val body: Block) data class ParsedForStatement(val forVariable: ForVariable, val iterable: Node) sealed class ForVariable @@ -30,11 +30,13 @@ data class ParsedDotName(val head: Node, val tail: Node) data class ParsedFunctionCall(val function: Node, val args: List) data class ParsedComparison(val cases: List) data class PrimitiveComparison(val left: Node, val op: Delimiter, val right: Node) +data class ParsedDecorator(val name: Name, val arguments: InvocationArguments? = null) fun parseFunctionDefinition(node: FunctionDefinition): ParsedFunctionDefinition? { val name = (node.children().first { it is Name } ?: return null) as Name val body = (node.children().find { it is Block } ?: return null) as Block - return ParsedFunctionDefinition(name, body) + val decoratorNodes = node.children().filterIsInstance().firstOrNull() + return ParsedFunctionDefinition(name, body, parseDecorators(decoratorNodes)) } fun parseClassDefinition(node: ClassDefinition): ParsedClassDefinition? { @@ -189,4 +191,23 @@ fun parseComparison(node: Comparison): ParsedComparison { primitives.add(PrimitiveComparison(children[i], children[i + 1] as Delimiter, children[i + 2])) } return ParsedComparison(primitives) -} \ No newline at end of file +} + +fun parseDecorators(decoratorNodes: Decorators?): List { + return decoratorNodes?.children()?.let { + val decoratorIndexes = + it.mapIndexedNotNull { index, node -> if (node is Delimiter && node.tokenType == PythonConstants.TokenType.AT) index + 1 else null } + decoratorIndexes.mapNotNull { index -> + val decorator = it[index] + when (decorator) { + is Name -> ParsedDecorator(decorator) + is FunctionCall -> { + val name = decorator.children().filterIsInstance().first() + val args = decorator.children().filterIsInstance().first() + ParsedDecorator(name, args) + } + else -> null + } + } + } ?: emptyList() +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/visitor/hints/HintCollector.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/visitor/hints/HintCollector.kt index 2e0adb4482..06cbbf57ba 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/visitor/hints/HintCollector.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/visitor/hints/HintCollector.kt @@ -3,6 +3,7 @@ package org.utbot.python.newtyping.ast.visitor.hints import org.parsers.python.Node import org.parsers.python.PythonConstants import org.parsers.python.ast.* +import org.utbot.python.PythonMethod import org.utbot.python.newtyping.* import org.utbot.python.newtyping.ast.* import org.utbot.python.newtyping.ast.visitor.Collector @@ -15,14 +16,14 @@ import org.utbot.python.newtyping.mypy.GlobalNamesStorage import java.util.* class HintCollector( - private val function: PythonFunctionDefinition, + private val function: PythonMethod, private val storage: PythonTypeHintsStorage, private val mypyTypes: Map, UtType>, private val globalNamesStorage: GlobalNamesStorage, private val moduleOfSources: String ) : Collector() { private val parameterToNode: Map = - (function.meta.args.map { it.name } zip function.type.arguments).associate { + (function.argumentsNames zip function.methodType.arguments).associate { it.first to HintCollectorNode(it.second) } private val astNodeToHintCollectorNode: MutableMap = mutableMapOf() @@ -30,7 +31,7 @@ class HintCollector( private val blockStack = Stack() init { - val argNames = function.meta.args.map { it.name } + val argNames = function.argumentsNames assert(argNames.all { it != "" }) identificationToNode[null] = mutableMapOf() argNames.forEach { @@ -45,7 +46,7 @@ class HintCollector( if (!allNodes.contains(it.value)) collectAllNodes(it.value, allNodes) } - result = HintCollectorResult(parameterToNode, function.type, allNodes) + result = HintCollectorResult(parameterToNode, function.methodType, allNodes) } private fun collectAllNodes(cur: HintCollectorNode, visited: MutableSet) { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/visitor/hints/Protocols.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/visitor/hints/Protocols.kt index 157881d647..2c2e888487 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/visitor/hints/Protocols.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/ast/visitor/hints/Protocols.kt @@ -1,11 +1,5 @@ package org.utbot.python.newtyping.ast.visitor.hints -import org.utbot.python.newtyping.* -import org.utbot.python.newtyping.general.CompositeTypeCreator -import org.utbot.python.newtyping.general.FunctionTypeCreator -import org.utbot.python.newtyping.general.Name -import org.utbot.python.newtyping.general.UtType - enum class Operation(val method: String) { Add("__add__"), Sub("__sub__"), diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceProcessor.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceProcessor.kt index be372bde78..ebc37a76be 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceProcessor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/TypeInferenceProcessor.kt @@ -4,18 +4,32 @@ import kotlinx.coroutines.runBlocking import org.parsers.python.PythonParser import org.parsers.python.ast.ClassDefinition import org.parsers.python.ast.FunctionDefinition -import org.utbot.python.PythonMethod -import org.utbot.python.newtyping.* +import org.utbot.python.PythonBaseMethod +import org.utbot.python.newtyping.PythonFunctionDefinition +import org.utbot.python.newtyping.PythonTypeHintsStorage import org.utbot.python.newtyping.ast.parseClassDefinition import org.utbot.python.newtyping.ast.parseFunctionDefinition import org.utbot.python.newtyping.ast.visitor.Visitor import org.utbot.python.newtyping.ast.visitor.hints.HintCollector import org.utbot.python.newtyping.general.CompositeType import org.utbot.python.newtyping.general.UtType +import org.utbot.python.newtyping.getPythonAttributeByName import org.utbot.python.newtyping.inference.baseline.BaselineAlgorithm -import org.utbot.python.newtyping.mypy.* +import org.utbot.python.newtyping.inference.baseline.MethodAndVars +import org.utbot.python.newtyping.mypy.GlobalNamesStorage +import org.utbot.python.newtyping.mypy.MypyBuildDirectory +import org.utbot.python.newtyping.mypy.MypyInfoBuild +import org.utbot.python.newtyping.mypy.dropInitFile +import org.utbot.python.newtyping.mypy.getErrorNumber +import org.utbot.python.newtyping.mypy.readMypyAnnotationStorageAndInitialErrors +import org.utbot.python.newtyping.pythonTypeRepresentation import org.utbot.python.newtyping.utils.getOffsetLine -import org.utbot.python.utils.* +import org.utbot.python.utils.Cleaner +import org.utbot.python.utils.Fail +import org.utbot.python.utils.Optional +import org.utbot.python.utils.RequirementsUtils +import org.utbot.python.utils.Success +import org.utbot.python.utils.TemporaryFileManager import java.io.File import java.nio.file.Path import java.nio.file.Paths @@ -84,7 +98,7 @@ class TypeInferenceProcessor( } val namesStorage = GlobalNamesStorage(mypyBuild) val collector = - HintCollector(pythonMethod.definition, typeStorage, mypyExpressionTypes, namesStorage, moduleOfSourceFile) + HintCollector(pythonMethod, typeStorage, mypyExpressionTypes, namesStorage, moduleOfSourceFile) val visitor = Visitor(listOf(collector)) visitor.visit(pythonMethod.ast) @@ -92,7 +106,8 @@ class TypeInferenceProcessor( typeStorage, collector.result, pythonPath, - pythonMethod, + MethodAndVars(pythonMethod, ""), + emptyList(), directoriesForSysPath, moduleOfSourceFile, namesInModule, @@ -103,7 +118,6 @@ class TypeInferenceProcessor( getOffsetLine(sourceFileContent, pythonMethod.ast.endOffset) ), mypyBuild.buildRoot.configFile, - "", dMypyTimeout = null ) @@ -122,7 +136,7 @@ class TypeInferenceProcessor( } } - private fun getPythonMethod(mypyInfoBuild: MypyInfoBuild, typeStorage: PythonTypeHintsStorage): Optional { + private fun getPythonMethod(mypyInfoBuild: MypyInfoBuild, typeStorage: PythonTypeHintsStorage): Optional { if (className == null) { val funcDef = parsedFile.children().firstNotNullOfOrNull { node -> val res = (node as? FunctionDefinition)?.let { parseFunctionDefinition(it) } @@ -133,7 +147,7 @@ class TypeInferenceProcessor( mypyInfoBuild.definitions[moduleOfSourceFile]!![functionName]!!.getUtBotDefinition() as? PythonFunctionDefinition ?: return Fail("$functionName is not a function") - val result = PythonMethod( + val result = PythonBaseMethod( functionName, path.toString(), null, @@ -160,7 +174,7 @@ class TypeInferenceProcessor( println(defOfFunc.type.pythonTypeRepresentation()) - val result = PythonMethod( + val result = PythonBaseMethod( functionName, path.toString(), typeOfClass, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt index fd1173ce17..ed2b293963 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/BaselineAlgorithm.kt @@ -2,12 +2,24 @@ package org.utbot.python.newtyping.inference.baseline import mu.KotlinLogging import org.utbot.python.PythonMethod -import org.utbot.python.newtyping.* -import org.utbot.python.newtyping.ast.visitor.hints.* +import org.utbot.python.newtyping.PythonTypeHintsStorage +import org.utbot.python.newtyping.ast.visitor.hints.EdgeSource +import org.utbot.python.newtyping.ast.visitor.hints.HintCollectorNode +import org.utbot.python.newtyping.ast.visitor.hints.HintCollectorResult +import org.utbot.python.newtyping.ast.visitor.hints.HintEdgeWithBound +import org.utbot.python.newtyping.createPythonUnionType import org.utbot.python.newtyping.general.FunctionType import org.utbot.python.newtyping.general.UtType -import org.utbot.python.newtyping.inference.* +import org.utbot.python.newtyping.inference.InferredTypeFeedback +import org.utbot.python.newtyping.inference.InvalidTypeFeedback +import org.utbot.python.newtyping.inference.SuccessFeedback +import org.utbot.python.newtyping.inference.TypeInferenceAlgorithm +import org.utbot.python.newtyping.inference.addEdge +import org.utbot.python.newtyping.inference.collectBoundsFromComponent +import org.utbot.python.newtyping.inference.visitNodesByReverseEdges import org.utbot.python.newtyping.mypy.checkSuggestedSignatureWithDMypy +import org.utbot.python.newtyping.pythonTypeRepresentation +import org.utbot.python.newtyping.typesAreEqual import org.utbot.python.newtyping.utils.weightedRandom import org.utbot.python.utils.TemporaryFileManager import java.io.File @@ -23,25 +35,35 @@ private val EDGES_TO_LINK = listOf( private val logger = KotlinLogging.logger {} +data class MethodAndVars( + val method: PythonMethod, + val additionalVars: String, +) + class BaselineAlgorithm( private val storage: PythonTypeHintsStorage, - private val hintCollectorResult: HintCollectorResult, + hintCollectorResult: HintCollectorResult, private val pythonPath: String, - private val pythonMethodCopy: PythonMethod, + private val pythonMethod: MethodAndVars, + methodModifications: List = emptyList(), private val directoriesForSysPath: Set, private val moduleToImport: String, private val namesInModule: Collection, private val initialErrorNumber: Int, private val configFile: File, - private val additionalVars: String, private val randomTypeFrequency: Int = 0, private val dMypyTimeout: Long? ) : TypeInferenceAlgorithm() { private val random = Random(0) + private val initialPythonMethod: PythonMethod = pythonMethod.method + private val generalRating = createGeneralTypeRating(hintCollectorResult, storage) - private val initialState = getInitialState(hintCollectorResult, generalRating) - private val states: MutableList = mutableListOf(initialState) + private val initialStates = methodModifications.ifEmpty { listOf(pythonMethod) } .map { + getInitialState(hintCollectorResult, generalRating, it.method.argumentsNames, it.method.methodType, it.additionalVars) + } + private val initialState = initialStates.first() + private val states: MutableList = initialStates.toMutableList() private val fileForMypyRuns = TemporaryFileManager.assignTemporaryFile(tag = "mypy.py") private var iterationCounter = 0 private var randomTypeCounter = 0 @@ -49,7 +71,7 @@ class BaselineAlgorithm( private val simpleTypes = simplestTypes(storage) private val mixtureType = createPythonUnionType(simpleTypes) - private val openedStates: MutableMap> = mutableMapOf() + private val expandedStates: MutableMap> = mutableMapOf() private val statistic: MutableMap = mutableMapOf() private val checkedSignatures: MutableSet = mutableSetOf() @@ -60,7 +82,7 @@ class BaselineAlgorithm( val newState = expandState(state, storage, state.anyNodes.map { mixtureType }) if (newState != null) { logger.info("Random type: ${newState.signature.pythonTypeRepresentation()}") - openedStates[newState.signature] = newState to state + expandedStates[newState.signature] = newState to state return newState.signature } return null @@ -95,40 +117,53 @@ class BaselineAlgorithm( val newState = expandState(state, storage) if (newState != null) { logger.info("Checking new state ${newState.signature.pythonTypeRepresentation()}") - if (checkSignature(newState.signature as FunctionType, fileForMypyRuns, configFile)) { + if (checkSignature(newState.signature as FunctionType, newState.additionalVars, fileForMypyRuns, configFile)) { logger.debug("Found new state!") - openedStates[newState.signature] = newState to state + expandedStates[newState.signature] = newState to state return newState.signature } } else if (state.anyNodes.isEmpty()) { if (state.signature in checkedSignatures) { + logger.debug("Good type ${state.signature.pythonTypeRepresentation()}") return state.signature } - logger.info("Checking ${state.signature.pythonTypeRepresentation()}") - if (checkSignature(state.signature as FunctionType, fileForMypyRuns, configFile)) { + logger.debug("Checking ${state.signature.pythonTypeRepresentation()}") + if (checkSignature(state.signature as FunctionType, state.additionalVars, fileForMypyRuns, configFile)) { + logger.debug("${state.signature.pythonTypeRepresentation()} is good") checkedSignatures.add(state.signature) return state.signature } else { states.remove(state) } } else { + logger.debug("Remove ${state.signature.pythonTypeRepresentation()} because of any nodes") states.remove(state) } return expandState() } fun feedbackState(signature: UtType, feedback: InferredTypeFeedback) { - val stateInfo = openedStates[signature] + val stateInfo = expandedStates[signature] + val lauded = statistic[signature] != 0 if (stateInfo != null) { val (newState, parent) = stateInfo - when (feedback) { - SuccessFeedback -> { + when { + feedback is SuccessFeedback || lauded -> { states.add(newState) parent.children += 1 } - InvalidTypeFeedback -> {} + + feedback is InvalidTypeFeedback -> { + states.remove(newState) + } + } + expandedStates.remove(signature) + } else if (feedback is InvalidTypeFeedback && !lauded) { + initialStates.forEach { + if (typesAreEqual(signature, it.signature)) { + states.remove(it) + } } - openedStates.remove(signature) } } @@ -161,7 +196,7 @@ class BaselineAlgorithm( annotationHandler(initialState.signature) } logger.info("Checking ${newState.signature.pythonTypeRepresentation()}") - if (checkSignature(newState.signature as FunctionType, fileForMypyRuns, configFile)) { + if (checkSignature(newState.signature as FunctionType, newState.additionalVars, fileForMypyRuns, configFile)) { logger.debug("Found new state!") when (annotationHandler(newState.signature)) { SuccessFeedback -> { @@ -179,13 +214,10 @@ class BaselineAlgorithm( return iterationCounter } - private fun checkSignature(signature: FunctionType, fileForMypyRuns: File, configFile: File): Boolean { - pythonMethodCopy.definition = PythonFunctionDefinition( - pythonMethodCopy.definition.meta, - signature - ) + private fun checkSignature(signature: FunctionType, newAdditionalVars: String, fileForMypyRuns: File, configFile: File): Boolean { + val methodCopy = initialPythonMethod.makeCopyWithNewType(signature) return checkSuggestedSignatureWithDMypy( - pythonMethodCopy, + methodCopy, directoriesForSysPath, moduleToImport, namesInModule, @@ -193,7 +225,7 @@ class BaselineAlgorithm( pythonPath, configFile, initialErrorNumber, - additionalVars, + newAdditionalVars, timeout = dMypyTimeout ) } @@ -205,10 +237,12 @@ class BaselineAlgorithm( private fun getInitialState( hintCollectorResult: HintCollectorResult, - generalRating: List + generalRating: List, + paramNames: List, + methodType: FunctionType, + additionalVars: String = "", ): BaselineAlgorithmState { - val paramNames = pythonMethodCopy.arguments.map { it.name } - val root = PartialTypeNode(hintCollectorResult.initialSignature, true) + val root = PartialTypeNode(methodType, true) val allNodes: MutableSet = mutableSetOf(root) val argumentRootNodes = paramNames.map { hintCollectorResult.parameterToNode[it]!! } argumentRootNodes.forEachIndexed { index, node -> @@ -224,10 +258,10 @@ class BaselineAlgorithm( ) addEdge(edge) } - return BaselineAlgorithmState(allNodes, generalRating, storage) + return BaselineAlgorithmState(allNodes, generalRating, storage, additionalVars) } fun laudType(type: FunctionType) { statistic[type] = statistic[type]?.plus(1) ?: 1 } -} \ No newline at end of file +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/StateExpansion.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/StateExpansion.kt index 71a49943c3..59b0e88268 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/StateExpansion.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/StateExpansion.kt @@ -5,8 +5,7 @@ import org.utbot.python.newtyping.general.UtType import org.utbot.python.newtyping.inference.addEdge import org.utbot.python.newtyping.pythonAnnotationParameters import org.utbot.python.newtyping.pythonDescription -import java.util.LinkedList -import java.util.Queue +import java.util.* fun expandState(state: BaselineAlgorithmState, typeStorage: PythonTypeHintsStorage): BaselineAlgorithmState? { if (state.anyNodes.isEmpty()) @@ -48,7 +47,7 @@ private fun expandNodes( addEdge(newEdge) } } - return BaselineAlgorithmState(newNodeMap.values.toSet() + allNewNodes, generalRating, storage) + return BaselineAlgorithmState(newNodeMap.values.toSet() + allNewNodes, generalRating, storage, state.additionalVars) } private fun expansionBFS( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/Structures.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/Structures.kt index 6c78170a00..97338bf590 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/Structures.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/inference/baseline/Structures.kt @@ -31,7 +31,8 @@ class BaselineAlgorithmEdge( class BaselineAlgorithmState( val nodes: Set, val generalRating: List, - typeStorage: PythonTypeHintsStorage + typeStorage: PythonTypeHintsStorage, + val additionalVars: String = "", ) { val signature: UtType get() = nodes.find { it.isRoot }!!.partialType diff --git a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/mypy/RunDMypy.kt b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/mypy/RunDMypy.kt index 882df30812..aec2fa1cac 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/newtyping/mypy/RunDMypy.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/newtyping/mypy/RunDMypy.kt @@ -41,7 +41,7 @@ fun checkSuggestedSignatureWithDMypy( timeout: Long? = null ): Boolean { val annotationMap = - (method.definition.meta.args.map { it.name } zip method.definition.type.arguments).associate { + (method.argumentsNames zip method.methodType.arguments).associate { Pair(it.first, it.second) } val mypyCode = generateMypyCheckCode(method, annotationMap, directoriesForSysPath, moduleToImport, namesInModule, additionalVars) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt index 77b3354b58..8d40d19a5d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/TestGenerationLimitManager.kt @@ -9,7 +9,7 @@ class TestGenerationLimitManager( var executions: Int = 150, var invalidExecutions: Int = 10, var cacheNodeExecutions: Int = 20, - var fakeNodeExecutions: Int = 1, + var fakeNodeExecutions: Int = 40, var missedLines: Int? = null, val isRootManager: Boolean = false, ) { @@ -39,6 +39,10 @@ class TestGenerationLimitManager( fakeNodeExecutions -= 1 } + fun restartFakeNode() { + fakeNodeExecutions = initFakeNodeExecutions + } + fun isCancelled(): Boolean { return mode.isCancelled(this) } @@ -77,3 +81,9 @@ object ExecutionWithTimoutMode : LimitManagerMode { return ExecutionMode.isCancelled(manager) || TimeoutMode.isCancelled(manager) } } + +object FakeWithTimeoutMode : LimitManagerMode { + override fun isCancelled(manager: TestGenerationLimitManager): Boolean { + return manager.fakeNodeExecutions <= 0 || TimeoutMode.isCancelled(manager) + } +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/TimeoutUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/TimeoutUtils.kt new file mode 100644 index 0000000000..416e05f447 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/TimeoutUtils.kt @@ -0,0 +1,31 @@ +package org.utbot.python.utils + +import java.text.SimpleDateFormat +import java.util.* +import kotlin.math.max + +fun separateUntil(until: Long, currentIndex: Int, itemsCount: Int): Long { + return when (val itemsLeft = itemsCount - currentIndex) { + 0 -> 0 + 1 -> until + else -> { + val now = System.currentTimeMillis() + max((until - now) / itemsLeft + now, 0) + } + } +} + +fun separateTimeout(timeout: Long, itemsCount: Int): Long { + return when (itemsCount) { + 0 -> 0 + else -> { + timeout / itemsCount + } + } +} + +fun Long.convertToTime(): String { + val date = Date(this) + val format = SimpleDateFormat("HH:mm:ss.SSS") + return format.format(date) +} \ No newline at end of file From 88ef3332da53c22c0db702a71502c7753176439c Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Tue, 12 Dec 2023 16:57:29 +0300 Subject: [PATCH 02/10] Added option for turning off symbolic or fuzzing --- .../python/sbft/SbftGenerateTestsCommand.kt | 13 +++--- .../python/PythonTestGenerationConfig.kt | 11 +++++ .../utbot/python/engine/GlobalPythonEngine.kt | 46 +++++++++++-------- 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt index bd873cc00f..e9e8016f35 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt @@ -8,6 +8,7 @@ import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.options.required import com.github.ajalt.clikt.parameters.options.split import com.github.ajalt.clikt.parameters.types.choice +import com.github.ajalt.clikt.parameters.types.enum import com.github.ajalt.clikt.parameters.types.long import mu.KotlinLogging import org.parsers.python.PythonParser @@ -16,12 +17,7 @@ import org.utbot.cli.language.python.findCurrentPythonModule import org.utbot.cli.language.python.toAbsolutePath import org.utbot.cli.language.python.writeToFileAndSave import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour -import org.utbot.python.MypyConfig -import org.utbot.python.PythonMethodHeader -import org.utbot.python.PythonTestGenerationConfig -import org.utbot.python.PythonTestGenerationProcessor -import org.utbot.python.TestFileInformation -import org.utbot.python.UsvmConfig +import org.utbot.python.* import org.utbot.python.code.PythonCode import org.utbot.python.coverage.CoverageOutputFormat import org.utbot.python.coverage.PythonCoverageMode @@ -115,6 +111,10 @@ class SbftGenerateTestsCommand : CliktCommand( ) .flag(default = false) + private val searchMode by option("--mode") + .enum { it.name } + .default(InputSearchMode.BOTH) + private val javaCmd by option( "--java-cmd", help = "(required) Path to Java command (ONLY FOR USVM)." @@ -273,6 +273,7 @@ class SbftGenerateTestsCommand : CliktCommand( prohibitedExceptions = if (prohibitedExceptions == listOf("-")) emptyList() else prohibitedExceptions, checkUsvm = checkUsvm, doNotGenerateStateAssertions = doNotGenerateStateAssertions, + inputSearchMode = searchMode ) val processor = SbftCliProcessor(config) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt index 41d493021e..481c6da86d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt @@ -34,6 +34,7 @@ class PythonTestGenerationConfig( val prohibitedExceptions: List = defaultProhibitedExceptions, val checkUsvm: Boolean = false, val doNotGenerateStateAssertions: Boolean = false, + val inputSearchMode: InputSearchMode = InputSearchMode.BOTH ) { companion object { val defaultProhibitedExceptions = listOf( @@ -60,4 +61,14 @@ data class UsvmConfig( val usvmJavaCmd = "${System.getProperty("java.home")}/bin/java" val defaultConfig = UsvmConfig(usvmJavaCmd, usvmDirectory) } +} + +@Suppress("unused") +enum class InputSearchMode( + val fuzzingEnabled: Boolean, + val symbolicEnabled: Boolean +) { + FUZZING(true, false), + SYMBOLIC(false, true), + BOTH(true, true) } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt index ae1982ac8c..30a9957327 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt @@ -105,26 +105,36 @@ class GlobalPythonEngine( } fun run() { - val fuzzing = thread( - start = true, - isDaemon = false, - name = "Fuzzer" - ) { - logger.info { " >>>>>>> Start fuzzer >>>>>>> " } - runFuzzing() - logger.info { " <<<<<<< Finish fuzzer <<<<<<< " } + val threads = mutableListOf() + if (configuration.inputSearchMode.fuzzingEnabled) { + val fuzzing = thread( + start = true, + isDaemon = false, + name = "Fuzzer" + ) { + logger.info { " >>>>>>> Start fuzzer >>>>>>> " } + runFuzzing() + logger.info { " <<<<<<< Finish fuzzer <<<<<<< " } + } + threads.add(fuzzing) + } else { + logger.info { "Fuzzing disabled." } } - val symbolic = thread( - start = true, - isDaemon = false, - name = "Symbolic" - ) { - logger.info { " ------- Start symbolic ------- " } - runSymbolic() - logger.info { " ======= Finish symbolic ======= " } + if (configuration.inputSearchMode.symbolicEnabled) { + val symbolic = thread( + start = true, + isDaemon = false, + name = "Symbolic" + ) { + logger.info { " ------- Start symbolic ------- " } + runSymbolic() + logger.info { " ======= Finish symbolic ======= " } + } + threads.add(symbolic) + } else { + logger.info { "Symbolic disabled." } } - fuzzing.join() - symbolic.join() + threads.forEach { it.join() } } fun debugUsvmRun() { From 8cadc906ae5f61addb1897e672e9a36967a55b0b Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Wed, 27 Dec 2023 15:27:33 +0300 Subject: [PATCH 03/10] Added option --only-toplevel --- .../language/python/sbft/SbftGenerateTestsCommand.kt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt index e9e8016f35..6d948a5e13 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt @@ -128,11 +128,20 @@ class SbftGenerateTestsCommand : CliktCommand( private val checkUsvm by option("--check-usvm", help = "Check usvm (ONLY FOR USVM).") .flag(default = false) + private val onlyToplevel by option( + "--only-toplevel" + ) + .flag(default = false) + private fun getPythonMethods(): List> { val parsedModule = PythonParser(sourceFileContent).Module() val topLevelFunctions = PythonCode.getTopLevelFunctions(parsedModule) - val topLevelClasses = PythonCode.getTopLevelClasses(parsedModule) + val topLevelClasses = + if (onlyToplevel) + emptyList() + else + PythonCode.getTopLevelClasses(parsedModule) val functions = topLevelFunctions .mapNotNull { parseFunctionDefinition(it) } From c808651243acb066f9abdfe0e2d847d240943e59 Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Thu, 1 Feb 2024 16:12:50 +0300 Subject: [PATCH 04/10] Take token from env in utbot-junit-contest --- utbot-junit-contest/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utbot-junit-contest/build.gradle b/utbot-junit-contest/build.gradle index e32d9c84c6..4d2afca976 100644 --- a/utbot-junit-contest/build.gradle +++ b/utbot-junit-contest/build.gradle @@ -10,8 +10,8 @@ repositories { maven { url = uri('https://maven.pkg.github.com/UnitTestBot/usvm') credentials { - username = project.githubActor // System.getenv("GITHUB_ACTOR") - password = project.githubToken // System.getenv("GITHUB_TOKEN") + username = System.getenv("GITHUB_ACTOR") + password = System.getenv("GITHUB_TOKEN") } } } From f889bd62f40d208892cce1c2ddb27f20b5db76ac Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Thu, 29 Feb 2024 13:58:27 +0300 Subject: [PATCH 05/10] update of usvm-python-runner --- gradle.properties | 2 ++ utbot-cli-python/build.gradle | 2 ++ .../python/sbft/SbftGenerateTestsCommand.kt | 5 +++++ .../src/main/resources/settings.properties | 2 +- utbot-junit-contest/build.gradle | 13 ------------- utbot-python/build.gradle.kts | 4 +++- .../org/utbot/python/PythonTestGenerationConfig.kt | 4 +++- .../org/utbot/python/engine/GlobalPythonEngine.kt | 3 ++- 8 files changed, 18 insertions(+), 17 deletions(-) diff --git a/gradle.properties b/gradle.properties index 6a9bfa25f4..2d724a9ec2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -135,3 +135,5 @@ org.gradle.parallel=true org.gradle.caching=true # there is no need to override the option below because parallel execution is disabled by --no-parallel org.gradle.workers.max=8 + +usvmPythonRunnerHash=0b8aff0 \ No newline at end of file diff --git a/utbot-cli-python/build.gradle b/utbot-cli-python/build.gradle index 350684a5ba..c0ebc2392d 100644 --- a/utbot-cli-python/build.gradle +++ b/utbot-cli-python/build.gradle @@ -24,6 +24,8 @@ dependencies { implementation group: 'com.github.ajalt.clikt', name: 'clikt', version: cliktVersion implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: log4j2Version implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: log4j2Version + + implementation group: 'org.usvm', name: 'usvm-python-commons', version: usvmPythonRunnerHash } processResources { diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt index 6d948a5e13..3061e1d3b5 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt @@ -12,6 +12,7 @@ import com.github.ajalt.clikt.parameters.types.enum import com.github.ajalt.clikt.parameters.types.long import mu.KotlinLogging import org.parsers.python.PythonParser +import org.usvm.python.ps.PyPathSelectorType import org.utbot.cli.language.python.CliRequirementsInstaller import org.utbot.cli.language.python.findCurrentPythonModule import org.utbot.cli.language.python.toAbsolutePath @@ -115,6 +116,10 @@ class SbftGenerateTestsCommand : CliktCommand( .enum { it.name } .default(InputSearchMode.BOTH) + private val pathSelector by option("--path-selector") + .enum { it.name } + .default(PyPathSelectorType.BaselinePriorityDfs) + private val javaCmd by option( "--java-cmd", help = "(required) Path to Java command (ONLY FOR USVM)." diff --git a/utbot-intellij-main/src/main/resources/settings.properties b/utbot-intellij-main/src/main/resources/settings.properties index 17328d9a5f..84bc852a2e 100644 --- a/utbot-intellij-main/src/main/resources/settings.properties +++ b/utbot-intellij-main/src/main/resources/settings.properties @@ -1,4 +1,4 @@ -# Copyright (c) 2023 utbot.org +# Copyright (c) 2024 utbot.org # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/utbot-junit-contest/build.gradle b/utbot-junit-contest/build.gradle index 4d2afca976..39dbf2a1c5 100644 --- a/utbot-junit-contest/build.gradle +++ b/utbot-junit-contest/build.gradle @@ -3,19 +3,6 @@ plugins { } apply plugin: 'jacoco' -repositories { - mavenLocal() - mavenCentral() - maven { url 'https://jitpack.io' } - maven { - url = uri('https://maven.pkg.github.com/UnitTestBot/usvm') - credentials { - username = System.getenv("GITHUB_ACTOR") - password = System.getenv("GITHUB_TOKEN") - } - } -} - configurations { fetchInstrumentationJar approximations diff --git a/utbot-python/build.gradle.kts b/utbot-python/build.gradle.kts index 3d0e0a3cbd..70996bbb23 100644 --- a/utbot-python/build.gradle.kts +++ b/utbot-python/build.gradle.kts @@ -3,6 +3,7 @@ val kotlinLoggingVersion: String? by rootProject val apacheCommonsTextVersion: String? by rootProject val jacksonVersion: String? by rootProject val log4j2Version: String? by rootProject +val usvmPythonRunnerHash: String by rootProject dependencies { api(project(":utbot-fuzzing")) @@ -10,7 +11,8 @@ dependencies { api(project(":utbot-python-parser")) api(project(":utbot-python-types")) api(project(":utbot-python-executor")) - api("org.usvm:usvm-python-runner:b087c5c") + api("org.usvm:usvm-python-runner:${usvmPythonRunnerHash}") + api("org.usvm:usvm-python-commons:${usvmPythonRunnerHash}") testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1") implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt index 481c6da86d..caf9cbb3b8 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonTestGenerationConfig.kt @@ -1,5 +1,6 @@ package org.utbot.python +import org.usvm.python.ps.PyPathSelectorType import org.utbot.framework.codegen.domain.RuntimeExceptionTestsBehaviour import org.utbot.framework.codegen.domain.TestFramework import org.utbot.python.coverage.CoverageOutputFormat @@ -34,7 +35,8 @@ class PythonTestGenerationConfig( val prohibitedExceptions: List = defaultProhibitedExceptions, val checkUsvm: Boolean = false, val doNotGenerateStateAssertions: Boolean = false, - val inputSearchMode: InputSearchMode = InputSearchMode.BOTH + val inputSearchMode: InputSearchMode = InputSearchMode.BOTH, + val pathSelector: PyPathSelectorType = PyPathSelectorType.BaselinePriorityDfs ) { companion object { val defaultProhibitedExceptions = listOf( diff --git a/utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt index 30a9957327..2eb96935a0 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/engine/GlobalPythonEngine.kt @@ -65,7 +65,8 @@ class GlobalPythonEngine( configuration.usvmConfig.javaCmd, mypyConfig.mypyBuildDirectory.root.canonicalPath, configuration.sysPathDirectories, - extractVenvConfig(configuration.pythonPath) + extractVenvConfig(configuration.pythonPath), + configuration.pathSelector ) val runner = SymbolicExecutionEvaluator( method, From a07e3be74a1427c21264785ff9e1671292def74d Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Thu, 29 Feb 2024 14:11:28 +0300 Subject: [PATCH 06/10] Fixed --path-selector parameter --- .../utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt index 3061e1d3b5..f3a365baf7 100644 --- a/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt +++ b/utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/sbft/SbftGenerateTestsCommand.kt @@ -287,7 +287,8 @@ class SbftGenerateTestsCommand : CliktCommand( prohibitedExceptions = if (prohibitedExceptions == listOf("-")) emptyList() else prohibitedExceptions, checkUsvm = checkUsvm, doNotGenerateStateAssertions = doNotGenerateStateAssertions, - inputSearchMode = searchMode + inputSearchMode = searchMode, + pathSelector = pathSelector ) val processor = SbftCliProcessor(config) From 5f1d78e3463aa4d7411f7bd8325e365d9d1f7c10 Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Fri, 1 Mar 2024 13:53:25 +0300 Subject: [PATCH 07/10] updated usvm --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 2d724a9ec2..c497c0bfab 100644 --- a/gradle.properties +++ b/gradle.properties @@ -136,4 +136,4 @@ org.gradle.caching=true # there is no need to override the option below because parallel execution is disabled by --no-parallel org.gradle.workers.max=8 -usvmPythonRunnerHash=0b8aff0 \ No newline at end of file +usvmPythonRunnerHash=086702c From 97f8a3542636f2c39d1fb53bd6c4ae9d7402b9d6 Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Sun, 3 Mar 2024 22:32:58 +0300 Subject: [PATCH 08/10] updated usvm --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index c497c0bfab..3647349c38 100644 --- a/gradle.properties +++ b/gradle.properties @@ -136,4 +136,4 @@ org.gradle.caching=true # there is no need to override the option below because parallel execution is disabled by --no-parallel org.gradle.workers.max=8 -usvmPythonRunnerHash=086702c +usvmPythonRunnerHash=431a073 From d446203a179f0af41cc1882a4645c07b56ac9f90 Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Mon, 11 Mar 2024 12:35:32 +0300 Subject: [PATCH 09/10] updated usvm --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 3647349c38..997b853332 100644 --- a/gradle.properties +++ b/gradle.properties @@ -136,4 +136,4 @@ org.gradle.caching=true # there is no need to override the option below because parallel execution is disabled by --no-parallel org.gradle.workers.max=8 -usvmPythonRunnerHash=431a073 +usvmPythonRunnerHash=c45af8f From d5f11c669d718435581a74c696ec8ebb09b15ee0 Mon Sep 17 00:00:00 2001 From: Ekaterina Tochilina Date: Thu, 23 May 2024 11:46:59 +0300 Subject: [PATCH 10/10] updated usvm --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 997b853332..f1b7aca2f3 100644 --- a/gradle.properties +++ b/gradle.properties @@ -136,4 +136,4 @@ org.gradle.caching=true # there is no need to override the option below because parallel execution is disabled by --no-parallel org.gradle.workers.max=8 -usvmPythonRunnerHash=c45af8f +usvmPythonRunnerHash=f30709f