fixes

weaviate · Jan 23, 2025 · 72ec7a1 · 72ec7a1
1 parent cc98abe
commit 72ec7a1
Show file tree

Hide file tree

Showing 31 changed files with 370 additions and 253 deletions.
diff --git a/...esult-postprocessing/create-radar-plot.py → experimental-results/create-radar-plot.py b/...esult-postprocessing/create-radar-plot.py → experimental-results/create-radar-plot.py
diff --git a/...-results/Llama-3.1-8B-Instruct-Turbo.json → ...s/legacy/Llama-3.1-8B-Instruct-Turbo.json b/...-results/Llama-3.1-8B-Instruct-Turbo.json → ...s/legacy/Llama-3.1-8B-Instruct-Turbo.json
diff --git a/experimental-results/claude-3-5-sonnet.json → ...tal-results/legacy/claude-3-5-sonnet.json b/experimental-results/claude-3-5-sonnet.json → ...tal-results/legacy/claude-3-5-sonnet.json
diff --git a/experimental-results/command-r-plus.json → ...mental-results/legacy/command-r-plus.json b/experimental-results/command-r-plus.json → ...mental-results/legacy/command-r-plus.json
diff --git a/experimental-results/command-r7b.json → experimental-results/legacy/command-r7b.json b/experimental-results/command-r7b.json → experimental-results/legacy/command-r7b.json
diff --git a/experimental-results/gemini-1.5-pro.json → ...mental-results/legacy/gemini-1.5-pro.json b/experimental-results/gemini-1.5-pro.json → ...mental-results/legacy/gemini-1.5-pro.json
diff --git a/...imental-results/gemini-2.0-flash-exp.json → ...-results/legacy/gemini-2.0-flash-exp.json b/...imental-results/gemini-2.0-flash-exp.json → ...-results/legacy/gemini-2.0-flash-exp.json
diff --git a/experimental-results/gpt-4o-mini.json → experimental-results/legacy/gpt-4o-mini.json b/experimental-results/gpt-4o-mini.json → experimental-results/legacy/gpt-4o-mini.json
diff --git a/experimental-results/gpt-4o.json → experimental-results/legacy/gpt-4o.json b/experimental-results/gpt-4o.json → experimental-results/legacy/gpt-4o.json
diff --git a/...ental-results/visualization-py/output.png → experimental-results/output.png b/...ental-results/visualization-py/output.png → experimental-results/output.png
diff --git a/...ts/result-postprocessing/print-queries.py → experimental-results/print-queries.py b/...ts/result-postprocessing/print-queries.py → experimental-results/print-queries.py
diff --git a/...sults/result-postprocessing/radar-plot.js → experimental-results/radar-plot.js b/...sults/result-postprocessing/radar-plot.js → experimental-results/radar-plot.js
diff --git a/...al-results/visualization-py/radar-plot.py → experimental-results/radar-plot.py b/...al-results/visualization-py/radar-plot.py → experimental-results/radar-plot.py
diff --git a/experimental-results/readme.md b/experimental-results/readme.md
@@ -1,25 +1,64 @@
-# Experimental Results
-
-```python
-class QueryPredictionResult(BaseModel):
-    query_index: int
-    database_schema_index: int
-    natural_language_query: str
-    ground_truth_query: WeaviateQueryWithSchema
-    predicted_query: Optional[WeaviateQuery]
-    ast_score: float
-    error: Optional[str]
-
-class ExperimentSummary(BaseModel):
-    timestamp: str
-    model_name: str
-    generate_with_models: bool
-    total_queries: int
-    successful_predictions: int
-    failed_predictions: int
-    average_ast_score: float
-    per_schema_scores: Dict[int, float]
-    detailed_results: List[QueryPredictionResult]
-```
-
-![Weaviate Gorilla](../visuals/weaviate-gorillas/gorilla-96.png)
+# Experiment Results
+
+## Overall Performance Comparison
+
+\begin{table}[h]
+\centering
+\begin{tabular}{|l|r|r|}
+\hline
+\textbf{Metric} & \textbf{GPT-4o} & \textbf{GPT-4o-mini} \\
+\hline
+Total Queries & 315 & 315 \\
+Successful Predictions & 304 & 308 \\
+Failed Predictions & 11 & 7 \\
+Average AST Score & 85.66\% & 83.43\% \\
+\hline
+\end{tabular}
+\caption{Overall Performance Metrics}
+\label{tab:overall-performance}
+\end{table}
+
+## Per Schema Performance
+
+\begin{table}[h]
+\centering
+\begin{tabular}{|l|r|r|}
+\hline
+\textbf{Schema} & \textbf{GPT-4o} & \textbf{GPT-4o-mini} \\
+\hline
+Schema 0 & 87.97\% & 84.14\% \\
+Schema 1 & 85.59\% & 84.10\% \\
+Schema 2 & 85.08\% & 82.30\% \\
+Schema 3 & 81.45\% & 82.23\% \\
+Schema 4 & 82.62\% & 79.18\% \\
+\hline
+\end{tabular}
+\caption{Performance Across Different Schemas}
+\label{tab:schema-performance}
+\end{table}
+
+## Component Analysis
+
+\begin{table}[h]
+\centering
+\begin{tabular}{|l|r|r|r|}
+\hline
+\textbf{Component Type} & \textbf{Sample Size} & \textbf{GPT-4o} & \textbf{GPT-4o-mini} \\
+\hline
+Search Queries & 160 & 76.77\% & 72.48\% \\
+Integer Filters & 80 & 79.28\% & 76.31\% \\
+Text Filters & 80 & 84.53\% & 85.16\% \\
+Boolean Filters & 80 & 91.44\% & 88.13\% \\
+Integer Aggregations & 80 & 82.38\% & 82.69\% \\
+Text Aggregations & 80 & 83.16\% & 78.78\% \\
+Boolean Aggregations & 80 & 87.03\% & 84.59\% \\
+GroupBy Operations & 160 & 83.53\% & 80.03\% \\
+\hline
+\end{tabular}
+\caption{Performance Analysis by Component Type}
+\label{tab:component-analysis}
+\end{table}
+
+# Latex
+
+![Weaviate Gorilla](../../visuals/weaviate-gorillas/gorilla-118.png)
diff --git a/experimental-results/result-postprocessing/ast-by-number-of-arguments.py b/experimental-results/result-postprocessing/ast-by-number-of-arguments.py
diff --git a/experimental-results/result-postprocessing/ast-test-postprocessor.py b/experimental-results/result-postprocessing/ast-test-postprocessor.py
diff --git a/experimental-results/result-postprocessing/readme.md b/experimental-results/result-postprocessing/readme.md