Feature/improve heuristics (#83)

* refactor similarity checks in ConnectionGenerator * minor refactoring regarding CorefAgent * update gitignore * Squashed 'tests/src/test/resources/benchmark/' content from commit 099cab6 git-subtree-dir: tests/src/test/resources/benchmark git-subtree-split: 099cab6 * add subtree for benchmarks Added the subtree itself that points to ArDoCo/Benchmark and is set to tests/src/test/resources/benchmark. Added a script to update the subtree. Adapted the tests to use the slightly changed location. Also created some packages for the architecture and integration tests for a more clear overview. * improve debug output for TracelinksIT * Squashed 'tests/src/test/resources/benchmark/' changes from 099cab6..6ce7247 6ce7247 fix goldstandards git-subtree-dir: tests/src/test/resources/benchmark git-subtree-split: 6ce7247 * update expected results * update expected results
ArDoCo · Feb 21, 2022 · a418d72 · a418d72
1 parent e20740f
commit a418d72
Show file tree

Hide file tree

Showing 52 changed files with 313 additions and 99 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,8 @@ tests/src/test/resources/config.properties
 
 tmp*
 
+.vscode/*
+
 ### ignore log files
 *.log
 

diff --git a/...a/edu/kit/kastel/mcse/ardoco/core/connectiongenerator/agents/InstanceConnectionAgent.java b/...a/edu/kit/kastel/mcse/ardoco/core/connectiongenerator/agents/InstanceConnectionAgent.java
@@ -1,7 +1,6 @@
 /* Licensed under MIT 2021. */
 package edu.kit.kastel.mcse.ardoco.core.connectiongenerator.agents;
 
-import org.eclipse.collections.api.factory.Lists;
 import org.eclipse.collections.api.list.ImmutableList;
 import org.kohsuke.MetaInfServices;
 
@@ -68,18 +67,16 @@ private void findNamesOfModelInstancesInSupposedMappings() {
         for (IModelInstance i : modelState.getInstances()) {
             ImmutableList<IRecommendedInstance> mostLikelyRi = SimilarityUtils.getMostRecommendedInstancesToInstanceByReferences(i, ris);
 
-            ImmutableList<IRecommendedInstance> mostLikelyRiWithoutType = mostLikelyRi.select(ri -> !ri.getTypeMappings().isEmpty());
-            mostLikelyRiWithoutType.forEach(ml -> connectionState.addToLinks(ml, i, probabilityWithoutType));
-            mostLikelyRi.forEach(ml -> connectionState.addToLinks(ml, i, probability));
+            for (var ri : mostLikelyRi) {
+                var riProbability = ri.getTypeMappings().isEmpty() ? probabilityWithoutType : probability;
+                connectionState.addToLinks(ri, i, riProbability);
+            }
         }
     }
 
     private void createLinksForEqualOrSimilarRecommendedInstances() {
         for (var ri : recommendationState.getRecommendedInstances()) {
-            var name = ri.getName();
-            var nameList = Lists.immutable.with(name.split(" "));
-            var sameInstances = modelState.getInstances()
-                    .select(i -> i.getLongestName().equalsIgnoreCase(name) || SimilarityUtils.areWordsOfListsSimilar(i.getNames(), nameList));
+            var sameInstances = modelState.getInstances().select(i -> SimilarityUtils.isRecommendedInstanceSimilarToModelInstance(ri, i));
             sameInstances.forEach(i -> connectionState.addToLinks(ri, i, probability));
         }
     }

diff --git a/...se/ardoco/core/connectiongenerator/extractors/ExtractionDependentOccurrenceExtractor.java b/...se/ardoco/core/connectiongenerator/extractors/ExtractionDependentOccurrenceExtractor.java
@@ -1,7 +1,6 @@
 /* Licensed under MIT 2021. */
 package edu.kit.kastel.mcse.ardoco.core.connectiongenerator.extractors;
 
-import org.eclipse.collections.api.factory.Lists;
 import org.kohsuke.MetaInfServices;
 
 import edu.kit.kastel.mcse.ardoco.core.common.Configuration;
@@ -71,6 +70,7 @@ public ConnectionExtractor create(ITextState textState, IModelState modelExtract
 
     @Override
     public void exec(IWord n) {
+        // TODO revisit and check if we want to check something different than only words as well
         searchForName(n);
         searchForType(n);
     }
@@ -82,8 +82,7 @@ public void exec(IWord n) {
      * @param word the node to check
      */
     private void searchForName(IWord word) {
-        boolean instanceNameIsSimilar = modelState.getInstances()
-                .anySatisfy(i -> SimilarityUtils.areWordsOfListsSimilar(i.getNames(), Lists.immutable.with(word.getText())));
+        boolean instanceNameIsSimilar = modelState.getInstances().anySatisfy(i -> SimilarityUtils.isWordSimilarToModelInstance(word, i));
         if (instanceNameIsSimilar) {
             textState.addName(word, probability);
         }
@@ -97,8 +96,7 @@ private void searchForName(IWord word) {
      * @param word the node to check
      */
     private void searchForType(IWord word) {
-        boolean instanceTypeIsSimilar = modelState.getInstances()
-                .anySatisfy(i -> SimilarityUtils.areWordsOfListsSimilar(i.getTypes(), Lists.immutable.with(word.getText())));
+        boolean instanceTypeIsSimilar = modelState.getInstances().anySatisfy(i -> SimilarityUtils.isWordSimilarToModelInstanceType(word, i));
         if (instanceTypeIsSimilar) {
             textState.addType(word, probability);
         }

diff --git a/tests/src/test/java/edu/kit/kastel/mcse/ardoco/core/tests/EvaluationResults.java b/tests/src/test/java/edu/kit/kastel/mcse/ardoco/core/tests/EvaluationResults.java
@@ -6,7 +6,7 @@
 import java.util.Locale;
 import java.util.Objects;
 
-class EvaluationResults {
+public class EvaluationResults {
     public double precision;
     public double recall;
     public double f1;

diff --git a/tests/src/test/java/edu/kit/kastel/mcse/ardoco/core/tests/Project.java b/tests/src/test/java/edu/kit/kastel/mcse/ardoco/core/tests/Project.java
@@ -18,35 +18,35 @@
 
 public enum Project {
     MEDIASTORE(//
-            "src/test/resources/mediastore/mediastore.owl", //
-            "src/test/resources/mediastore/mediastore_w_text.owl", //
-            "src/test/resources/mediastore/mediastore.txt", //
-            "src/test/resources/mediastore/goldstandard.csv", //
-            new EvaluationResults(.943, .606, .738), //
-            new EvaluationResults(.99, .606, .755)//
+            "src/test/resources/benchmark/mediastore/mediastore.owl", //
+            "src/test/resources/benchmark/mediastore/mediastore_w_text.owl", //
+            "src/test/resources/benchmark/mediastore/mediastore.txt", //
+            "src/test/resources/benchmark/mediastore/goldstandard.csv", //
+            new EvaluationResults(1.0, .620, .765), //
+            new EvaluationResults(.99, .586, .739)//
     ), //
     TEAMMATES( //
-            "src/test/resources/teammates/teammates.owl", //
-            "src/test/resources/teammates/teammates_w_text.owl", //
-            "src/test/resources/teammates/teammates.txt", //
-            "src/test/resources/teammates/goldstandard.csv", //
-            new EvaluationResults(.877, .877, .877), //
-            new EvaluationResults(.882, .828, .854)//
+            "src/test/resources/benchmark/teammates/teammates.owl", //
+            "src/test/resources/benchmark/teammates/teammates_w_text.owl", //
+            "src/test/resources/benchmark/teammates/teammates.txt", //
+            "src/test/resources/benchmark/teammates/goldstandard.csv", //
+            new EvaluationResults(.889, .879, .884), //
+            new EvaluationResults(.895, .830, .861)//
     ), //
     TEASTORE( //
-            "src/test/resources/teastore/teastore.owl", //
-            "src/test/resources/teastore/teastore_w_text.owl", //
-            "src/test/resources/teastore/teastore.txt", //
-            "src/test/resources/teastore/goldstandard.csv", //
+            "src/test/resources/benchmark/teastore/teastore.owl", //
+            "src/test/resources/benchmark/teastore/teastore_w_text.owl", //
+            "src/test/resources/benchmark/teastore/teastore.txt", //
+            "src/test/resources/benchmark/teastore/goldstandard.csv", //
             new EvaluationResults(.99, .713, .832), //
             new EvaluationResults(.99, .713, .832)),
     BIGBLUEBUTTON( //
-            "src/test/resources/bigbluebutton/bbb.owl", //
-            "src/test/resources/bigbluebutton/bbb_w_text.owl", //
-            "src/test/resources/bigbluebutton/bigbluebutton.txt", //
-            "src/test/resources/bigbluebutton/goldstandard.csv", //
-            new EvaluationResults(.734, .817, .773), //
-            new EvaluationResults(.734, .817, .773));
+            "src/test/resources/benchmark/bigbluebutton/bbb.owl", //
+            "src/test/resources/benchmark/bigbluebutton/bbb_w_text.owl", //
+            "src/test/resources/benchmark/bigbluebutton/bigbluebutton.txt", //
+            "src/test/resources/benchmark/bigbluebutton/goldstandard.csv", //
+            new EvaluationResults(.856, .839, .847), //
+            new EvaluationResults(.856, .839, .847));
 
     private final String model;
     private final String textOntology;

diff --git a/tests/src/test/java/edu/kit/kastel/mcse/ardoco/core/tests/TestUtil.java b/tests/src/test/java/edu/kit/kastel/mcse/ardoco/core/tests/TestUtil.java
@@ -13,13 +13,13 @@
 
 import org.eclipse.collections.api.factory.Lists;
 
-class TestUtil {
+public class TestUtil {
 
     private TestUtil() {
         throw new IllegalAccessError("This constructor should not be called!");
     }
 
-    protected static EvaluationResults compare(Collection<String> traceLinks, Collection<String> goldStandard) {
+    public static EvaluationResults compare(Collection<String> traceLinks, Collection<String> goldStandard) {
         Set<String> distinctTraceLinks = new HashSet<>(traceLinks);
         Set<String> distinctGoldStandard = new HashSet<>(goldStandard);
 

diff --git a/...e/ardoco/core/tests/ArchitectureTest.java → .../tests/architecture/ArchitectureTest.java b/...e/ardoco/core/tests/ArchitectureTest.java → .../tests/architecture/ArchitectureTest.java
@@ -1,5 +1,5 @@
 /* Licensed under MIT 2021. */
-package edu.kit.kastel.mcse.ardoco.core.tests;
+package edu.kit.kastel.mcse.ardoco.core.tests.architecture;
 
 import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.classes;
 import static com.tngtech.archunit.library.Architectures.layeredArchitecture;

diff --git a/...se/ardoco/core/tests/InconsistencyIT.java → ...re/tests/integration/InconsistencyIT.java b/...se/ardoco/core/tests/InconsistencyIT.java → ...re/tests/integration/InconsistencyIT.java
@@ -1,5 +1,5 @@
 /* Licensed under MIT 2021. */
-package edu.kit.kastel.mcse.ardoco.core.tests;
+package edu.kit.kastel.mcse.ardoco.core.tests.integration;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -16,6 +16,7 @@
 import org.junit.jupiter.params.provider.EnumSource;
 
 import edu.kit.kastel.mcse.ardoco.core.model.IModelConnector;
+import edu.kit.kastel.mcse.ardoco.core.tests.Project;
 import edu.kit.kastel.mcse.ardoco.core.tests.inconsistencies.eval.AbstractEvalStrategy;
 import edu.kit.kastel.mcse.ardoco.core.tests.inconsistencies.eval.EvaluationResult;
 import edu.kit.kastel.mcse.ardoco.core.tests.inconsistencies.eval.GoldStandard;

diff --git a/...o/core/tests/InstanceRelationAgentIT.java → .../integration/InstanceRelationAgentIT.java b/...o/core/tests/InstanceRelationAgentIT.java → .../integration/InstanceRelationAgentIT.java
@@ -1,5 +1,5 @@
 /* Licensed under MIT 2021. */
-package edu.kit.kastel.mcse.ardoco.core.tests;
+package edu.kit.kastel.mcse.ardoco.core.tests.integration;
 
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -39,8 +39,8 @@
 
 class InstanceRelationAgentIT {
 
-    private static final String TEXT = "src/test/resources/mediastore/mediastore.txt";
-    private static final String MODEL = "src/test/resources/mediastore/mediastore.owl";
+    private static final String TEXT = "src/test/resources/benchmark/mediastore/mediastore.txt";
+    private static final String MODEL = "src/test/resources/benchmark/mediastore/mediastore.owl";
 
     @BeforeEach
     void beforeEach() {

diff --git a/...mcse/ardoco/core/tests/PerformanceIT.java → ...core/tests/integration/PerformanceIT.java b/...mcse/ardoco/core/tests/PerformanceIT.java → ...core/tests/integration/PerformanceIT.java
@@ -1,5 +1,5 @@
 /* Licensed under MIT 2021. */
-package edu.kit.kastel.mcse.ardoco.core.tests;
+package edu.kit.kastel.mcse.ardoco.core.tests.integration;
 
 import java.io.File;
 import java.time.Duration;
@@ -73,14 +73,14 @@ private Duration measureExecution() {
 
     private void prepareOntology() {
         inputText = null;
-        var inputFilePath = String.format("src/test/resources/%s/%s_w_text.owl", name, name);
+        var inputFilePath = String.format("src/test/resources/benchmark/%s/%s_w_text.owl", name, name);
         inputModel = new File(inputFilePath);
     }
 
     private void prepareText() {
-        var inputTextPath = String.format("src/test/resources/%s/%s.txt", name, name);
+        var inputTextPath = String.format("src/test/resources/benchmark/%s/%s.txt", name, name);
         inputText = new File(inputTextPath);
-        var inputFilePath = String.format("src/test/resources/%s/%s.owl", name, name);
+        var inputFilePath = String.format("src/test/resources/benchmark/%s/%s.owl", name, name);
         inputModel = new File(inputFilePath);
     }
 

diff --git a/...el/mcse/ardoco/core/tests/PipelineIT.java → ...co/core/tests/integration/PipelineIT.java b/...el/mcse/ardoco/core/tests/PipelineIT.java → ...co/core/tests/integration/PipelineIT.java
@@ -1,5 +1,5 @@
 /* Licensed under MIT 2021. */
-package edu.kit.kastel.mcse.ardoco.core.tests;
+package edu.kit.kastel.mcse.ardoco.core.tests.integration;
 
 import java.io.File;
 import java.io.FileInputStream;

diff --git a/...kastel/mcse/ardoco/core/tests/TextIT.java → ...ardoco/core/tests/integration/TextIT.java b/...kastel/mcse/ardoco/core/tests/TextIT.java → ...ardoco/core/tests/integration/TextIT.java
@@ -1,5 +1,5 @@
 /* Licensed under MIT 2021. */
-package edu.kit.kastel.mcse.ardoco.core.tests;
+package edu.kit.kastel.mcse.ardoco.core.tests.integration;
 
 import java.io.File;
 import java.io.FileInputStream;
@@ -24,8 +24,8 @@
 
 class TextIT {
 
-    private static final String TEXT = "src/test/resources/teammates/teammates.txt";
-    private static final String MODEL_OWL = "src/test/resources/teammates/teammates.owl";
+    private static final String TEXT = "src/test/resources/benchmark/teammates/teammates.txt";
+    private static final String MODEL_OWL = "src/test/resources/benchmark/teammates/teammates.owl";
     private String tmpOwlFile;
 
     @Disabled("Only for local testing")
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,6 +4,8 @@ tests/src/test/resources/config.properties @@
     tmp*
+    .vscode/*
     ### ignore log files
     *.log
@@ Expand Down @@