Align doc output format with repro scripts (castorini#2485)

docs/reproduce/msmarco-v1-passage.html

@@ -218,18 +218,18 @@ <h1 class="mb-3">MS MARCO V1 Passage</h1>
   -threads 16 \
   -index msmarco-v1-passage \
   -topics dl19-passage \
-  -output runs/run.msmarco.bm25-default.dl19.txt \
+  -output runs/run.msmarco.bm25.dl19.txt \
   -hits 1000 -bm25
 </code></pre></blockquote>
 Evaluation commands:
 
   <blockquote class="mycode">
 <pre><code>tools/eval/trec_eval.9.0.4/trec_eval -c -l 2 -m map dl19-passage \
-  runs/run.msmarco.bm25-default.dl19.txt
+  runs/run.msmarco.bm25.dl19.txt
 tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 dl19-passage \
-  runs/run.msmarco.bm25-default.dl19.txt
+  runs/run.msmarco.bm25.dl19.txt
 tools/eval/trec_eval.9.0.4/trec_eval -c -l 2 -m recall.1000 dl19-passage \
-  runs/run.msmarco.bm25-default.dl19.txt
+  runs/run.msmarco.bm25.dl19.txt
 </code></pre>
   </blockquote>
 
@@ -242,18 +242,18 @@ <h1 class="mb-3">MS MARCO V1 Passage</h1>
   -threads 16 \
   -index msmarco-v1-passage \
   -topics dl20-passage \
-  -output runs/run.msmarco.bm25-default.dl20.txt \
+  -output runs/run.msmarco.bm25.dl20.txt \
   -hits 1000 -bm25
 </code></pre></blockquote>
 Evaluation commands:
 
   <blockquote class="mycode">
 <pre><code>tools/eval/trec_eval.9.0.4/trec_eval -c -l 2 -m map dl20-passage \
-  runs/run.msmarco.bm25-default.dl20.txt
+  runs/run.msmarco.bm25.dl20.txt
 tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 dl20-passage \
-  runs/run.msmarco.bm25-default.dl20.txt
+  runs/run.msmarco.bm25.dl20.txt
 tools/eval/trec_eval.9.0.4/trec_eval -c -l 2 -m recall.1000 dl20-passage \
-  runs/run.msmarco.bm25-default.dl20.txt
+  runs/run.msmarco.bm25.dl20.txt
 </code></pre>
   </blockquote>
 
@@ -266,16 +266,16 @@ <h1 class="mb-3">MS MARCO V1 Passage</h1>
   -threads 16 \
   -index msmarco-v1-passage \
   -topics msmarco-v1-passage.dev \
-  -output runs/run.msmarco.bm25-default.dev.txt \
+  -output runs/run.msmarco.bm25.dev.txt \
   -hits 1000 -bm25
 </code></pre></blockquote>
 Evaluation commands:
 
   <blockquote class="mycode">
 <pre><code>tools/eval/trec_eval.9.0.4/trec_eval -c -M 10 -m recip_rank msmarco-passage.dev-subset \
-  runs/run.msmarco.bm25-default.dev.txt
+  runs/run.msmarco.bm25.dev.txt
 tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 msmarco-passage.dev-subset \
-  runs/run.msmarco.bm25-default.dev.txt
+  runs/run.msmarco.bm25.dev.txt
 </code></pre>
   </blockquote>
 

src/main/java/io/anserini/reproduce/RunBeir.java

@@ -24,7 +24,7 @@
 public class RunBeir {
 
   public static void main(String[] args) throws Exception {
-    RunRepro repro = new RunRepro("beir", new BeirMetricDefinitions());
+    RunRepro repro = new RunRepro("beir", new BeirMetricDefinitions(), false);
     repro.run();
   }
 

src/main/java/io/anserini/reproduce/RunMsMarco.java

@@ -17,9 +17,12 @@
 package io.anserini.reproduce;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.kohsuke.args4j.CmdLineException;
 import org.kohsuke.args4j.CmdLineParser;
@@ -32,14 +35,13 @@ public class RunMsMarco {
   public static class Args {
     @Option(name = "-options", usage = "Print information about options.")
     public Boolean options = false;
-    @Option(name = "-v", metaVar = "[int]", usage = "MsMarco Version (1/2), where 2 is for V2.1. Default 1.")
-    public int MsMarcoVersion = 1;
+    @Option(name = "-v", usage = "MsMarco Version (msmarco-v2.1 / msmarco-v1-passage). Default: msmarco-v1-passage.")
+    public String MsMarcoVersion = "msmarco-v1-passage";
   }
 
   public static void main(String[] args) throws Exception {
 
     // check for cmd option
-    String COLLECTION = "msmarco-v1-passage";
     Args MsMarcoArgs = new Args();
     CmdLineParser parser = new CmdLineParser(MsMarcoArgs, ParserProperties.defaults().withUsageWidth(120));
 
@@ -64,16 +66,14 @@ public static void main(String[] args) throws Exception {
 
       return;
     }
-    switch (MsMarcoArgs.MsMarcoVersion) {
-        case 2:
-            COLLECTION = "msmarco-v2.1-doc";
-            break;
-        default: // MsMarcoVersion == 1
-            COLLECTION = "msmarco-v1-passage";
-            break;
+
+    Set<String> allowedVersions = new HashSet<>(Arrays.asList("msmarco-v2.1", "msmarco-v1-passage"));
+    if (!allowedVersions.contains(MsMarcoArgs.MsMarcoVersion)) {
+        System.err.println("Invalid MsMarco version: " + MsMarcoArgs.MsMarcoVersion);
+        System.exit(1);
     }
 
-    RunRepro repro = new RunRepro(COLLECTION, new MsMarcoMetricDefinitions());
+    RunRepro repro = new RunRepro(MsMarcoArgs.MsMarcoVersion, new MsMarcoMetricDefinitions(), true);
     repro.run();
   }
 
@@ -141,7 +141,7 @@ public MsMarcoMetricDefinitions() {
       dl23PassageMetrics.put("R@1K", "-c -m recall.1000");
       msmarcoV2Passage.put("dl23-doc-msmarco-v2.1", dl23PassageMetrics);
 
-      metricDefinitions.put("msmarco-v2.1-doc", msmarcoV2Passage);
+      metricDefinitions.put("msmarco-v2.1", msmarcoV2Passage);
     }
   }
 

src/main/java/io/anserini/reproduce/RunRepro.java

@@ -40,11 +40,13 @@ public class RunRepro {
 
   private String COLLECTION;
   private TrecEvalMetricDefinitions metricDefinitions;
+  private boolean sortByCondition;
 
-  public RunRepro(String collection, TrecEvalMetricDefinitions metrics)
+  public RunRepro(String collection, TrecEvalMetricDefinitions metrics, boolean sortCondition)
       throws IOException, InterruptedException {
     COLLECTION = collection;
     metricDefinitions = metrics;
+    sortByCondition = sortCondition;
   }
 
   public void run() throws StreamReadException, DatabindException, IOException, InterruptedException, URISyntaxException {
@@ -64,8 +66,14 @@ public void run() throws StreamReadException, DatabindException, IOException, In
       for (Topic topic : condition.topics) {
         System.out.println("  - topic_key: " + topic.topic_key + "\n");
 
-        final String output = String.format("runs/run.%s.%s.%s.txt", COLLECTION, condition.name,
-            topic.topic_key);
+        final String output;
+        if (sortByCondition) {
+          output = String.format("runs/run.%s.%s.%s.txt", COLLECTION, condition.name,
+          topic.topic_key);
+        } else {
+          output = String.format("runs/run.%s.%s.%s.txt", COLLECTION, topic.topic_key,
+          condition.name);
+        }
 
         final String command = condition.command
             .replace("$fatjar", fatjarPath)
@@ -93,6 +101,9 @@ public void run() throws StreamReadException, DatabindException, IOException, In
         for (Map<String, Double> expected : topic.scores) {
           for (String metric : expected.keySet()) {
             String evalKey = topic.eval_key;
+            if (!evalCommands.get(evalKey).containsKey(metric)) {
+              continue; // skip metric unintended to test
+            }
             String evalCmd = "java -cp $fatjarPath trec_eval $metric $evalKey $output"
             .replace("$fatjarPath", fatjarPath)
             .replace("$metric", evalCommands.get(evalKey).get(metric))
@@ -113,8 +124,7 @@ public void run() throws StreamReadException, DatabindException, IOException, In
               if (delta > 0.00005) {
                 System.out.println(String.format("    %7s: %.4f %s expected %.4f", metric, score, FAIL,
                     expected.get(metric)));
-              }
-              else {
+              } else {
                 System.out.println(String.format("    %7s: %.4f [OK]", metric, score));
               }
             } else {

src/main/resources/reproduce/msmarco-v1-passage.yaml

@@ -1,5 +1,5 @@
 conditions:
-  - name: bm25-default
+  - name: bm25
     display: "BM25 (k1=0.9, b=0.4)"
     display_html: "BM25 (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)"
     display_row: ""
@@ -297,4 +297,4 @@ conditions:
         scores:
           - MAP: 0.5052
             nDCG@10: 0.7262
-            R@1K: 0.8616
+            R@1K: 0.8616

src/main/resources/reproduce/msmarco-v2.1-doc.yaml → src/main/resources/reproduce/msmarco-v2.1.yaml

@@ -1,5 +1,5 @@
 conditions:
-  - name: bm25
+  - name: doc
     display: "BM25 doc (k1=0.9, b=0.4)"
     display_html: "BM25 doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)"
     display_row: ""
@@ -37,7 +37,7 @@ conditions:
             nDCG@10: 0.2914
             R@100: 0.2604
             R@1K: 0.5383
-  - name: bm25-segmented
+  - name: doc-segmented
     display: "BM25 segmented doc (k1=0.9, b=0.4)"
     display_html: "BM25 segmented doc (<i>k<sub><small>1</small></sub></i>=0.9, <i>b</i>=0.4)"
     display_row: ""

src/test/java/io/anserini/doc/GenerateReproductionDocsTest.java

@@ -40,7 +40,7 @@ public class GenerateReproductionDocsTest {
   public final static String ROW_TEMPLATE_PATH = "src/main/resources/reproduce/msmarco_html_row_v1.template";
   public final static String COLLECTION = "msmarco-v1-passage";
   public final static String[] MODELS = {
-      "bm25-default",
+      "bm25",
       "splade-pp-ed.cached_q",
       "splade-pp-ed.onnx",
       "cos-dpr-distil.fp32.cached_q",