FFT: KNN

README.md

@@ -1,58 +1,28 @@
-Heatmap:
-https://matplotlib.org/stable/gallery/images_contours_and_fields/image_annotated_heatmap.html#sphx-glr-gallery-images-contours-and-fields-image-annotated-heatmap-py
+# Multimedia retrieval 
+This is a multimedia retrieval system, which takes a query shape and returns similar shapes from the database.
+Use alt and control with the mouse to move the shape. It uses Open3D to render the shapes.
 
-Embed in PyQt:
-https://github.com/isl-org/Open3D/discussions/4668
+## How to run the project
+The full application only works on Windows. I have verified with PyCharm that the script can run.
+Please install all the packages from the requirements.txt located on the same layer as this readme.
 
-Put qml into widget:
-https://stackoverflow.com/questions/59828470/how-use-qml-drawer-with-qt-widgets
-https://github.com/eyllanesc/stackoverflow/blob/master/questions/59828470/main.py
+Then run the main.py in src for the offline part.
+Or run main.py in the app folder for the GUI
 
-
-https://github.com/isl-org/Open3D/issues/2063
-
-Open3D github repository:
-
-## Compute bary-center of object
-Geometry.GetCenter()
-Geometry3D::ComputeCenter(points)
-
-```c++
-Eigen::Vector3d Geometry3D::ComputeCenter(
-        const std::vector<Eigen::Vector3d>& points) const {
-    Eigen::Vector3d center(0, 0, 0);
-    if (points.empty()) {
-        return center;
-    }
-    center = std::accumulate(points.begin(), points.end(), center);
-    center /= double(points.size());
-    return center;
-}
+Important: Run from the repository root like this. It will indicate with a warning if it is done incorrectly
+```commandline
+python3 src/main.py
 ```
 
-Camera viewpoint:
-https://github.com/isl-org/Open3D/issues/1553
-
-Faster iteration:
-https://numpy.org/doc/stable/reference/arrays.nditer.html
+or
 
-Random points:
-https://stackoverflow.com/questions/14262654/numpy-get-random-set-of-rows-from-2d-array
-
-Angle:
-https://stackoverflow.com/questions/35176451/python-code-to-calculate-angle-between-three-point-using-their-3d-coordinates
-
-Performance:
-https://mmas.github.io/python-image-processing-libraries-performance-opencv-scipy-scikit-image
-
-Alternatives for pywin32?:
-https://stackoverflow.com/questions/373020/finding-the-current-active-window-in-mac-os-x-using-python
-https://pypi.org/project/PyGetWindow/
-https://developer.apple.com/documentation/appkit/nsworkspace#//apple_ref/occ/instm/NSWorkspace/frontmostApplication
+```commandline
+python3 app/main.py
+```
 
-Sample points on mesh uniformly fast:
-https://github.com/isl-org/Open3D/issues/2062
+## Other notes
+Please be aware that querying takes a long time since the convex hull etc. is computed for the shapes.
+It didn't take more than 30 seconds on my system, but maybe some large shapes will slow it down substantially
 
-Relay keypresses to open3d:
-https://github.com/matplotlib/matplotlib/issues/707
-https://github.com/isl-org/Open3D/blob/master/cpp/open3d/visualization/visualizer/Visualizer.cpp
+The paths are printed to the command line after the query has been performed if it would take too long. 
+They are in the correct ordering and can then be used in the ply viewer tab.

app/util/font.py

@@ -5,9 +5,12 @@
 FONT: QFont = QFont(FONT_NAME)
 
 SMALL_FONT = QFont(FONT_NAME, pointSize=5)
+LARGE_FONT = QFont(FONT_NAME, pointSize=11)
 
 BOLD_FONT: QFont = QFont(FONT_NAME)
 BOLD_FONT.setBold(True)
+LARGE_BOLD_FONT = QFont(FONT_NAME, pointSize=11)
+LARGE_BOLD_FONT.setBold(True)
 
 ITALIC_FONT: QFont = QFont(FONT_NAME)
 ITALIC_FONT.setItalic(True)

app/widget/query_result_widget.py

@@ -3,6 +3,7 @@
 
 from app.widget.util import color_widget, create_header_label
 from app.widget.visualization_widget import VisualizationWidget
+from util.font import LARGE_BOLD_FONT
 
 
 class QueryResultWidget(QWidget):
@@ -16,6 +17,7 @@ def __init__(self, title, settings):
         # Load widget
         self.scene_widget = VisualizationWidget(self.settings)
         self.distance_label = create_header_label("Distance: -")
+        self.distance_label.setFont(LARGE_BOLD_FONT)
 
         # Capture scene widget into a window
         window = QWindow.fromWinId(self.scene_widget.hwnd)

app/widget/tab/query_tab_widget.py

@@ -2,6 +2,7 @@
 from PyQt6.QtGui import QWindow
 
 from app.widget.query_result_widget import QueryResultWidget
+from database.knn_querier import KNNQuerier
 from src.pipeline.feature_extractor.shape_properties_extractor import ShapePropertyExtractor
 from src.database.custom_querier import CustomQuerier
 from src.object.settings import Settings
@@ -18,19 +19,25 @@
 class QueryTabWidget(QWidget):
     RESULTS_PER_ROW = 6
 
-    def __init__(self):
+    def __init__(self, use_custom_querier: bool = True):
         super(QueryTabWidget, self).__init__()
         color_widget(self, [0, 255, 0])
 
         # Left panel
         self.settings: Settings = Settings()
         self.settings_widget = SettingsWidget(self.settings)
         self.pipeline = NormalizationPipeline()
-        # self.querier = DatabaseQuerier(os.path.join(DATABASE_REFINED_DIR, DATABASE_NORMALIZED_DESCRIPTORS_FILENAME))
-        self.querier = CustomQuerier(
-            os.path.join(DATABASE_NORMALIZED_DIR, DATABASE_NORMALIZED_DESCRIPTORS_FILENAME),
-            os.path.join(DATABASE_NORMALIZED_DIR, DATABASE_PROPERTIES_FILENAME)
-        )
+
+        if use_custom_querier:
+            self.querier = CustomQuerier(
+                os.path.join(DATABASE_NORMALIZED_DIR, DATABASE_NORMALIZED_DESCRIPTORS_FILENAME),
+                os.path.join(DATABASE_NORMALIZED_DIR, DATABASE_PROPERTIES_FILENAME)
+            )
+        else:
+            self.querier = KNNQuerier(
+                os.path.join(DATABASE_NORMALIZED_DIR, DATABASE_NORMALIZED_DESCRIPTORS_FILENAME),
+                os.path.join(DATABASE_NORMALIZED_DIR, DATABASE_PROPERTIES_FILENAME)
+            )
 
         # Load widget
         self.query_scene_widget = VisualizationWidget(self.settings)

app/widget/tab_widget.py

@@ -42,7 +42,8 @@ def __init__(self):
         self.features_table_widget = FeaturesTableTabWidget()
         self.descriptors_table_widget = DescriptorsTableTabWidget()
         self.normalized_descriptors_table_widget = NormalizedDescriptorsTableTabWidget()
-        self.query_widget = QueryTabWidget()
+        self.custom_query_widget = QueryTabWidget(True)
+        self.knn_query_widget = QueryTabWidget(False)
         self.tsne_widget = TsneCanvasTabWidget()
         self.performance_widget = PerformanceTabWidget()
 
@@ -53,7 +54,8 @@ def __init__(self):
         self.addTab(self.features_table_widget, "Features table")
         self.addTab(self.descriptors_table_widget, "Descriptors table")
         self.addTab(self.normalized_descriptors_table_widget, "Normalized descriptors table")
-        self.addTab(self.query_widget, "Query")
+        self.addTab(self.custom_query_widget, "Custom query")
+        self.addTab(self.knn_query_widget, "KNN query")
         self.addTab(self.tsne_widget, "t-SNE")
         self.addTab(self.performance_widget, "Performance")
 

data/database/.gitignore

@@ -0,0 +1 @@
+knn_distances.npy

src/database/custom_querier.py

@@ -75,6 +75,7 @@ def calc_distances_row(self, shape: Shape) -> ([str], [float]):
         # Sorted paths and distances
         sorted_paths = [path for path, _ in sorted_tuples]
         sorted_distances = [distance for _, distance in sorted_tuples]
+        print('paths of custom query in order:', sorted_paths)
 
         return sorted_paths, sorted_distances
 

src/database/knn_querier.py

@@ -2,59 +2,65 @@
 from copy import deepcopy
 from pynndescent import NNDescent
 
+from src.object.shape import Shape
 from src.database.reader import FeatureDatabaseReader
-from src.object.descriptors import Descriptors
 from src.pipeline.normalize_descriptors import compute_normalized_descriptor
 from src.util.configs import *
 
 
 class KNNQuerier:
-    def __init__(self, path: str):
-        """Parse descriptors and corresponding paths
+    def __init__(self, descriptors_path: str, properties_path: str):
+        """Parse descriptors, properties, and corresponding paths"""
+        descriptors = FeatureDatabaseReader.read_descriptors(descriptors_path)
+        properties = FeatureDatabaseReader.read_properties(properties_path)
 
-        :param path: path to the descriptor database to read from
-        """
-        self.descriptors = FeatureDatabaseReader.read_descriptors(path)
-
-        # Path did not exist
-        if len(self.descriptors) == 0:
+        # List of descriptors
+        if len(descriptors) == 0:
             logging.error('Querier empty')
             return
 
+        # Read descriptors, properties and append them in the same order to the lists
         self.paths = []
-        self.values = []
-        for identifier in self.descriptors:
+        self.feature_vectors = []
+        for identifier in descriptors.keys():
             self.paths.append(identifier)
-            self.values.append(self.descriptors[identifier])
+            descriptor_values = np.array(descriptors[identifier].to_list())
+            properties_values = np.array(properties[identifier].to_list()).reshape(-1)
+            feature_vector = np.concatenate([descriptor_values, properties_values])
+            self.feature_vectors.append(feature_vector)
+
+        # Multiply by weights
+        self.feature_vectors = np.array(self.feature_vectors) * KNN_WEIGHT_VECTOR
 
         self.index: NNDescent = None
 
-    def query_descriptor(self, descriptors: Descriptors) -> [(str, float)]:
+    def calc_distances_row(self, shape: Shape) -> ([str], [float]):
         """Query for shapes most similar to the given descriptors
 
-        :param descriptors: Descriptor information of the query shape
+        :param shape: Shape containing all the descriptors and properties
         :return: Paths and indices that are most similar to the given descriptors
         """
-        descriptors_copy = deepcopy(descriptors)
-        compute_normalized_descriptor(descriptors_copy)
-        return self.query_normalized_descriptor(descriptors_copy)
+        # Normalize descriptor
+        descriptor_copy = deepcopy(shape.descriptors)
+        compute_normalized_descriptor(descriptor_copy)
+        normalized_query_descriptors = np.array(descriptor_copy.to_list())
 
-    def query_normalized_descriptor(self, descriptors: Descriptors) -> [(str, float)]:
-        """Query for the descriptors once normalized
+        # Concat to one large feature vector of length 109.
+        histogram_properties = np.array(shape.properties.to_list()).reshape(-1)
+        feature_vector = np.concatenate([normalized_query_descriptors, histogram_properties])
+        feature_vector *= KNN_WEIGHT_VECTOR
 
-        :param descriptors: Descriptor information of the query shape
-        :return: Paths and indices that are most similar to the given descriptors
-        """
+        # Create query structure if it did not yet exist
         if self.index is None:
             logging.info('It is creating the NN structure, please wait for 20 seconds')
-            self.index = NNDescent(np.array(self.values)[:50], verbose=True)
+            self.index = NNDescent(np.array(self.feature_vectors), verbose=True)
             logging.info('Preparing the structure')
             self.index.prepare()
 
         # Descriptors to search for
-        descriptors = np.array(descriptors.to_list())
-        top_k_neighbour_indices, top_k_distances = self.index.query([descriptors], k=NR_RESULTS)
-        top_k_paths = np.array(self.values)[top_k_neighbour_indices]
+        top_k_neighbour_indices, top_k_distances = self.index.query([feature_vector], k=NR_RESULTS)
+        top_k_paths = list(np.array(self.paths)[top_k_neighbour_indices].reshape(-1))
+        print('paths of KNN in order:', top_k_paths)
 
         # Return top
-        return top_k_paths, top_k_neighbour_indices
+        return top_k_paths, list(top_k_distances.reshape(-1))