[WIP] Search

Cargo.toml

@@ -52,4 +52,5 @@ members = [
     "components/taxonomies",
     "components/templates",
     "components/utils",
+    "components/search",
 ]

components/front_matter/src/page.rs

@@ -62,6 +62,7 @@ fn fix_toml_dates(table: Map<String, Value>) -> Value {
 
 /// The front matter of every page
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(default)]
 pub struct PageFrontMatter {
     /// <title> of the page
     pub title: Option<String>,
@@ -96,10 +97,9 @@ pub struct PageFrontMatter {
     pub template: Option<String>,
     /// Whether the page is included in the search index
     /// Defaults to `true` but is only used if search if explicitly enabled in the config.
-    #[serde(default, skip_serializing)]
+    #[serde(skip_serializing)]
     pub in_search_index: bool,
     /// Any extra parameter present in the front matter
-    #[serde(default)]
     pub extra: Map<String, Value>,
 }
 

components/search/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "search"
+version = "0.1.0"
+authors = ["Vincent Prouillet <[email protected]>"]
+
+[dependencies]
+elasticlunr-rs = "1"
+ammonia = "1"
+lazy_static = "1"
+
+errors = { path = "../errors" }
+content = { path = "../content" }

components/search/src/lib.rs

@@ -0,0 +1,71 @@
+extern crate elasticlunr;
+#[macro_use]
+extern crate lazy_static;
+extern crate ammonia;
+
+extern crate errors;
+extern crate content;
+
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+
+use elasticlunr::Index;
+use content::Section;
+
+
+pub const ELASTICLUNR_JS: &'static str = include_str!("elasticlunr.min.js");
+
+lazy_static! {
+    static ref AMMONIA: ammonia::Builder<'static> = {
+        let mut clean_content = HashSet::new();
+        clean_content.insert("script");
+        clean_content.insert("style");
+        let mut builder = ammonia::Builder::new();
+        builder
+            .tags(HashSet::new())
+            .tag_attributes(HashMap::new())
+            .generic_attributes(HashSet::new())
+            .link_rel(None)
+            .allowed_classes(HashMap::new())
+            .clean_content_tags(clean_content);
+        builder
+    };
+}
+
+
+/// Returns the generated JSON index with all the documents of the site added
+/// TODO: is making `in_search_index` apply to subsections of a `false` section useful?
+pub fn build_index(sections: &HashMap<PathBuf, Section>) -> String {
+    let mut index = Index::new(&["title", "body"]);
+
+    for section in sections.values() {
+        add_section_to_index(&mut index, section);
+    }
+
+    index.to_json()
+}
+
+fn add_section_to_index(index: &mut Index, section: &Section) {
+    if !section.meta.in_search_index {
+        return;
+    }
+
+    // Don't index redirecting sections
+    if section.meta.redirect_to.is_none() {
+        index.add_doc(
+            &section.permalink,
+            &[&section.meta.title.clone().unwrap_or(String::new()), &AMMONIA.clean(&section.content).to_string()],
+        );
+    }
+
+    for page in &section.pages {
+        if !page.meta.in_search_index {
+            continue;
+        }
+
+        index.add_doc(
+            &page.permalink,
+            &[&page.meta.title.clone().unwrap_or(String::new()), &AMMONIA.clean(&page.content).to_string()],
+        );
+    }
+}

components/site/Cargo.toml

@@ -19,6 +19,7 @@ front_matter = { path = "../front_matter" }
 pagination = { path = "../pagination" }
 taxonomies = { path = "../taxonomies" }
 content = { path = "../content" }
+search = { path = "../search" }
 
 [dev-dependencies]
 tempdir = "0.3"

components/site/src/lib.rs

@@ -15,6 +15,7 @@ extern crate templates;
 extern crate pagination;
 extern crate taxonomies;
 extern crate content;
+extern crate search;
 
 #[cfg(test)]
 extern crate tempdir;
@@ -509,7 +510,32 @@ impl Site {
             self.compile_sass(&self.base_path)?;
         }
 
-        self.copy_static_directories()
+        self.copy_static_directories()?;
+
+        if self.config.build_search_index {
+            self.build_search_index()?;
+        }
+
+        Ok(())
+    }
+
+    pub fn build_search_index(&self) -> Result<()> {
+        // index first
+        create_file(
+            &self.output_path.join("search_index.js"),
+            &format!(
+                "window.searchIndex = {};",
+                search::build_index(&self.sections)
+            ),
+        )?;
+
+        // then elasticlunr.min.js
+        create_file(
+            &self.output_path.join("elasticlunr.min.js"),
+            search::ELASTICLUNR_JS,
+        )?;
+
+        Ok(())
     }
 
     pub fn compile_sass(&self, base_path: &Path) -> Result<()> {

components/site/tests/site.rs

@@ -449,6 +449,17 @@ fn can_build_rss_feed() {
 
 #[test]
 fn can_build_search_index() {
-    // TODO: generate an index somehow and check for correctness with
-    // another one
+    let mut path = env::current_dir().unwrap().parent().unwrap().parent().unwrap().to_path_buf();
+    path.push("test_site");
+    let mut site = Site::new(&path, "config.toml").unwrap();
+    site.load().unwrap();
+    site.config.build_search_index = true;
+    let tmp_dir = TempDir::new("example").expect("create temp dir");
+    let public = &tmp_dir.path().join("public");
+    site.set_output_path(&public);
+    site.build().unwrap();
+
+    assert!(Path::new(&public).exists());
+    assert!(file_exists!(public, "elasticlunr.min.js"));
+    assert!(file_exists!(public, "search_index.js"));
 }

docs/config.toml

@@ -6,6 +6,7 @@ compile_sass = true
 highlight_code = true
 insert_anchor_links = true
 highlight_theme = "kronuz"
+build_search_index = true
 
 [extra]
 author = "Vincent Prouillet"

docs/sass/_search.scss

@@ -0,0 +1,3 @@
+.search-results {
+  display: none;
+}

docs/sass/site.scss

@@ -16,3 +16,4 @@ $link-color: #007CBC;
 @import "index";
 @import "docs";
 @import "themes";
+@import "search";

docs/static/search.js

@@ -0,0 +1,60 @@
+function formatSearchResultHeader(term, count) {
+  if (count === 0) {
+    return "No search results for '" + term + "'.";
+  }
+
+  return count + " search result" + count > 1 ? "s" : "" + " for '" + term + "':";
+}
+
+function formatSearchResultItem(term, item) {
+  console.log(item);
+  return '<div class="search-results__item">'
+  + item
+  + '</div>';
+}
+
+function initSearch() {
+  var $searchInput = document.getElementById("search");
+  var $searchResults = document.querySelector(".search-results");
+  var $searchResultsHeader = document.querySelector(".search-results__headers");
+  var $searchResultsItems = document.querySelector(".search-results__items");
+
+  var options = {
+    bool: "AND",
+    expand: true,
+    teaser_word_count: 30,
+    limit_results: 30,
+    fields: {
+      title: {boost: 2},
+      body: {boost: 1},
+    }
+  };
+  var currentTerm = "";
+  var index = elasticlunr.Index.load(window.searchIndex);
+
+  $searchInput.addEventListener("keyup", function() {
+    var term = $searchInput.value.trim();
+    if (!index || term === "" || term === currentTerm) {
+      return;
+    }
+    $searchResults.style.display = term === "" ? "block" : "none";
+    $searchResultsItems.innerHTML = "";
+    var results = index.search(term, options);
+    currentTerm = term;
+    $searchResultsHeader.textContent = searchResultText(term, results.length);
+    for (var i = 0; i < results.length; i++) {
+      var item = document.createElement("li");
+      item.innerHTML = formatSearchResult(results[i], term);
+      $searchResultsItems.appendChild(item);
+    }
+  });
+}
+
+
+if (document.readyState === "complete" ||
+    (document.readyState !== "loading" && !document.documentElement.doScroll)
+) {
+  initSearch();
+} else {
+  document.addEventListener("DOMContentLoaded", initSearch);
+}

docs/templates/index.html

@@ -18,9 +18,15 @@
                 <a class="white" href="{{ get_url(path="./documentation/_index.md") }}" class="nav-link">Docs</a>
                 <a class="white" href="{{ get_url(path="./themes/_index.md") }}" class="nav-link">Themes</a>
                 <a class="white" href="https://github.com/Keats/gutenberg" class="nav-link">GitHub</a>
+                <input id="search" type="search" placeholder="Search the docs">
             </nav>
         </header>
 
+        <div class="search-results">
+            <h2 class="search-results__header"></h2>
+            <div class="search-results__items"></div>
+        </div>
+
         <div class="content {% block extra_content_class %}{% endblock extra_content_class %}">
             {% block content %}
                 <div class="hero">
@@ -93,5 +99,9 @@ <h2>Augmented Markdown</h2>
         <footer>
             ©2017-2018 — <a class="white" href="https://vincent.is">Vincent Prouillet</a> and <a class="white" href="https://github.com/Keats/gutenberg/graphs/contributors">contributors</a>
         </footer>
+
+        <script type="text/javascript" src="{{ get_url(path="elasticlunr.min.js", trailing_slash=false) }}"></script>
+        <script type="text/javascript" src="{{ get_url(path="search_index.js", trailing_slash=false) }}"></script>
+        <script type="text/javascript" src="{{ get_url(path="search.js", trailing_slash=false) }}"></script>
     </body>
 </html>