Add typeahead for syntax highlighting languages.

Modified composebox_typeahead.js to recognize the triple backtick
and tilde for code blocks, and added appropriate typeahead functions
in that file and in typeahead_helper.js.

Additionally, a new file pygments_data.js contains a dictionary of
the supported languages, mapping to relative popularity
rankings. These rankings determine the order of sort of the
languages in the typeahead.

This JavaScript file is actually in static/generated/pygments_data.js, as it
is generated by a Python script, tools/build_pymgents_data.py. This is
so that if Pygments adds support for new languages, the JavaScript file
will be updated appropriately. This python script uses a set of popularity
rankings defined in lang.json.

Corresponding unit tests were also added.

Fixes zulip#4111.

.eslintrc.json

@@ -23,6 +23,7 @@
         "attachments_ui": false,
         "csrf_token": false,
         "typeahead_helper": false,
+        "pygments_data": false,
         "popovers": false,
         "server_events": false,
         "ui": false,

.gitignore

@@ -18,6 +18,7 @@ coverage/
 /zproject/dev-secrets.conf
 static/js/bundle.js
 static/generated/emoji
+static/generated/pygments_data.js
 static/generated/github-contributors.json
 static/locale/language_options.json
 static/third/emoji-data

docs/dev-setup-non-vagrant.md

@@ -321,6 +321,7 @@ Now run these commands:
 ```
 ./tools/install-mypy
 ./tools/setup/emoji/build_emoji
+./tools/setup/build_pygments_data.py
 ./scripts/setup/generate_secrets.py --development
 if [ $(uname) = "OpenBSD" ]; then
     sudo cp ./puppet/zulip/files/postgresql/zulip_english.stop /var/postgresql/tsearch_data/
@@ -461,4 +462,3 @@ the results in your browser.
 Currently, the Docker workflow is substantially less convenient than
 the Vagrant workflow and less documented; please contribute to this
 guide and the Docker tooling if you are using Docker to develop Zulip!
-

frontend_tests/node_tests/composebox_typeahead.js

@@ -8,6 +8,9 @@ set_global('emoji', {emojis: emoji_list});
 set_global('stream_data', {subscribed_subs: function () {
     return stream_list;
 }});
+set_global('pygments_data', {langs:
+    {python: 0, javscript: 1, html: 2, css: 3},
+});
 
 global.stub_out_jquery();
 add_dependencies({
@@ -46,7 +49,7 @@ global.people.add({
     ct.split_at_cursor = function (word) { return [word, '']; };
 
     var begin_typehead_this = {options: {completions: {
-        emoji: true, mention: true, stream: true}}};
+        emoji: true, mention: true, stream: true, syntax: true}}};
 
     function assert_typeahead_equals(input, reference) {
         var returned = ct.compose_content_begins_typeahead.call(begin_typehead_this, input);
@@ -90,6 +93,17 @@ global.people.add({
     assert_typeahead_equals("test #", false);
     assert_typeahead_equals("test #D", stream_list);
     assert_typeahead_equals("#s", stream_list);
+
+    var lang_list = Object.keys(pygments_data.langs);
+    assert_typeahead_equals("``` ", false);
+    assert_typeahead_equals("test ``` py", false);
+    assert_typeahead_equals("test ```a", false);
+    assert_typeahead_equals("```b", lang_list);
+    assert_typeahead_equals("``c", false);
+    assert_typeahead_equals("``` d", lang_list);
+    assert_typeahead_equals("~~~e", lang_list);
+    assert_typeahead_equals("~~~ f", lang_list);
+    assert_typeahead_equals("test ~~~", false);
 }());
 
 (function test_tokenizing() {
@@ -102,6 +116,9 @@ global.people.add({
     assert.equal(ct.tokenize_compose_str("foo bar :smil"), ":smil");
     assert.equal(ct.tokenize_compose_str(":smil"), ":smil");
     assert.equal(ct.tokenize_compose_str("foo @alice sm"), "@alice sm");
+    assert.equal(ct.tokenize_compose_str("foo ```p"), "");
+    assert.equal(ct.tokenize_compose_str("``` py"), "``` py");
+    assert.equal(ct.tokenize_compose_str("foo``bar ~~~ py"), "");
 
     // The following cases are kinda judgment calls...
     assert.equal(ct.tokenize_compose_str(

frontend_tests/node_tests/typeahead_helper.js

@@ -43,6 +43,16 @@ assert.deepEqual(test_streams[2].name, "Derp"); // Less subscribers
 assert.deepEqual(test_streams[3].name, "Dev"); // Alphabetically last
 assert.deepEqual(test_streams[4].name, "dead"); // Inactive streams last
 
+set_global('pygments_data', {langs:
+    {python: 40, javscript: 50, php: 38, pascal: 29, perl: 22, css: 0},
+});
+
+var test_langs = ["pascal", "perl", "php", "python", "javascript"];
+test_langs = typeahead_helper.sort_languages(test_langs, "p");
+
+// Sort languages by matching first letter, and then by popularity
+assert.deepEqual(test_langs, ["python", "php", "pascal", "perl", "javascript"]);
+
 var matches = [
     {
         email: "[email protected]",

static/js/composebox_typeahead.js

@@ -48,6 +48,11 @@ function composebox_typeahead_highlighter(item) {
     return typeahead_helper.highlight_with_escaping(this.query, item);
 }
 
+function query_matches_language(query, lang) {
+    query = query.toLowerCase();
+    return lang.indexOf(query) !== -1;
+}
+
 function query_matches_person(query, person) {
     // Case-insensitive.
     query = query.toLowerCase();
@@ -212,6 +217,15 @@ exports.tokenize_compose_str = function (s) {
     while (i > min_i) {
         i -= 1;
         switch (s[i]) {
+            case '`':
+            case '~':
+                // Code block must start on a new line
+                if (i === 2) {
+                    return s.slice(0);
+                } else if (i > 2 && s[i-3] === "\n") {
+                    return s.slice(i-2);
+                }
+                break;
             case '#':
             case '@':
             case ':':
@@ -234,6 +248,29 @@ exports.compose_content_begins_typeahead = function (query) {
         return false;
     }
 
+    // Start syntax highlighting autocompleter if the first three characters are ```
+    var syntax_token = current_token.substring(0,3);
+    if (this.options.completions.syntax && (syntax_token === '```' || syntax_token === "~~~")) {
+        // Only autocomplete if user starts typing a language after ```
+        if (current_token.length === 3) {
+            return false;
+        }
+
+        // If the only input is a space, don't autocomplete
+        current_token = current_token.substring(3);
+        if (current_token === " ") {
+            return false;
+        }
+
+        // Trim the first whitespace if it is there
+        if (current_token[0] === " ") {
+            current_token = current_token.substring(1);
+        }
+        this.completing = 'syntax';
+        this.token = current_token;
+        return Object.keys(pygments_data.langs);
+    }
+
     // Only start the emoji autocompleter if : is directly after one
     // of the whitespace or punctuation chars we split on.
     if (this.options.completions.emoji && current_token[0] === ':') {
@@ -309,6 +346,8 @@ exports.content_highlighter = function (item) {
         return typeahead_helper.highlight_with_escaping(this.token, item_formatted);
     } else if (this.completing === 'stream') {
         return typeahead_helper.render_stream(this.token, item);
+    } else if (this.completing === 'syntax') {
+        return typeahead_helper.highlight_with_escaping(this.token, item);
     }
 };
 
@@ -332,18 +371,25 @@ exports.content_typeahead_selected = function (item) {
         beginning = (beginning.substring(0, beginning.length - this.token.length-1)
                 + '#**' + item.name + '** ');
         $(document).trigger('streamname_completed.zulip', {stream: item});
+    } else if (this.completing === 'syntax') {
+        rest = "\n" + beginning.substring(beginning.length - this.token.length - 4,
+                beginning.length - this.token.length).trim() + rest;
+        beginning = beginning.substring(0, beginning.length - this.token.length) + item + "\n";
     }
 
     // Keep the cursor after the newly inserted text, as Bootstrap will call textbox.change() to
     // overwrite the text in the textbox.
     setTimeout(function () {
         $('#new_message_content').caret(beginning.length, beginning.length);
+        // Also, trigger autosize to check if compose box needs to be resized.
+        compose.autosize_textarea();
     }, 0);
     return beginning + rest;
 };
 
 exports.initialize_compose_typeahead = function (selector, completions) {
-    completions = $.extend({mention: false, emoji: false, stream: false}, completions);
+    completions = $.extend(
+        {mention: false, emoji: false, stream: false, syntax: false}, completions);
 
     $(selector).typeahead({
         items: 5,
@@ -358,6 +404,8 @@ exports.initialize_compose_typeahead = function (selector, completions) {
                 return query_matches_person(this.token, item);
             } else if (this.completing === 'stream') {
                 return query_matches_stream(this.token, item);
+            } else if (this.completing === 'syntax') {
+                return query_matches_language(this.token, item);
             }
         },
         sorter: function (matches) {
@@ -368,6 +416,8 @@ exports.initialize_compose_typeahead = function (selector, completions) {
                                                         compose_state.stream_name());
             } else if (this.completing === 'stream') {
                 return typeahead_helper.sort_streams(matches, this.token);
+            } else if (this.completing === 'syntax') {
+                return typeahead_helper.sort_languages(matches, this.token);
             }
         },
         updater: exports.content_typeahead_selected,
@@ -493,7 +543,7 @@ exports.initialize = function () {
         stopAdvance: true, // Do not advance to the next field on a tab or enter
     });
 
-    exports.initialize_compose_typeahead("#new_message_content", {mention: true, emoji: true, stream: true});
+    exports.initialize_compose_typeahead("#new_message_content", {mention: true, emoji: true, stream: true, syntax: true});
 
     $( "#private_message_recipient" ).blur(function () {
         var val = $(this).val();

static/js/typeahead_helper.js

@@ -182,6 +182,24 @@ exports.sort_for_at_mentioning = function (objs, current_stream) {
     return subs_sorted.concat(non_subs_sorted);
 };
 
+exports.compare_by_popularity = function (lang_a, lang_b) {
+    var diff = pygments_data.langs[lang_b] - pygments_data.langs[lang_a];
+    if (diff !== 0) {
+        return diff;
+    }
+    return util.strcmp(lang_a, lang_b);
+};
+
+exports.sort_languages = function (matches, query) {
+    var results = prefix_sort(query, matches, function (x) { return x; });
+
+    // Languages that start with the query
+    results.matches = results.matches.sort(exports.compare_by_popularity);
+    // Languages that have the query somewhere in their name
+    results.rest = results.rest.sort(exports.compare_by_popularity);
+    return results.matches.concat(results.rest);
+};
+
 exports.sort_recipients = function (matches, query, current_stream) {
     var name_results =  prefix_sort(query, matches, function (x) { return x.full_name; });
     var email_results = prefix_sort(query, name_results.rest, function (x) { return x.email; });

tools/lib/provision.py

@@ -219,6 +219,7 @@ def main(options):
     run(["sudo", "chown", "%s:%s" % (user_id, user_id), EMOJI_CACHE_PATH])
     run(["tools/setup/emoji/download-emoji-data"])
     run(["tools/setup/emoji/build_emoji"])
+    run(["tools/setup/build_pygments_data.py"])
     run(["scripts/setup/generate_secrets.py", "--development"])
     run(["tools/update-authors-json", "--use-fixture"])
     if options.is_travis and not options.is_production_travis:

tools/setup/build_pygments_data.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+from pygments.lexers import get_all_lexers
+import json
+import os
+
+ZULIP_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../')
+DATA_PATH = os.path.join(ZULIP_PATH, 'tools', 'setup', 'lang.json')
+JS_PATH = os.path.join(ZULIP_PATH, 'static', 'generated', 'pygments_data.js')
+
+with open(DATA_PATH) as f:
+    langs = json.load(f)
+
+lexers = get_all_lexers()
+for lexer in lexers:
+    for name in lexer[1]:
+        if name not in langs:
+            langs[name] = 0
+
+template = '''var pygments_data = (function () {
+
+var exports = {};
+
+exports.langs = %s;
+
+return exports;
+
+}());
+if (typeof module !== 'undefined') {
+    module.exports = pygments_data;
+}''' % json.dumps(langs)
+
+with open(JS_PATH, 'w') as f:
+    f.write(template)

tools/setup/lang.json

@@ -0,0 +1,57 @@
+{
+    "abap": 27,
+    "ada": 25,
+    "awk": 1,
+    "bash": 7,
+    "c": 49,
+    "c#": 47,
+    "c++": 48,
+    "cobol": 26,
+    "cpp": 48,
+    "csharp": 47,
+    "css": 48,
+    "d": 29,
+    "dart": 28,
+    "delphi": 42,
+    "erlang": 10,
+    "fsharp": 19,
+    "go": 34,
+    "groovy": 13,
+    "haskell": 15,
+    "html": 30,
+    "java": 50,
+    "javascript": 51,
+    "js": 43,
+    "julia": 4,
+    "latex": 40,
+    "lisp": 18,
+    "lua": 22,
+    "mask": 2,
+    "math": 50,
+    "matlab": 33,
+    "mql": 9,
+    "mql4": 9,
+    "objective-c": 35,
+    "objectivec": 35,
+    "objectpascal": 42,
+    "pascal": 42,
+    "perl": 40,
+    "php": 44,
+    "pl": 40,
+    "prolog": 16,
+    "python": 46,
+    "quote": 50,
+    "r": 37,
+    "rb": 39,
+    "ruby": 39,
+    "rust": 8,
+    "sas": 30,
+    "scala": 21,
+    "scheme": 14,
+    "sql": 32,
+    "swift": 41,
+    "tex": 40,
+    "vb.net": 45,
+    "vbnet": 45,
+    "xml": 1
+}

tools/update-prod-static

@@ -45,6 +45,10 @@ subprocess.check_call(['./tools/setup/emoji/download-emoji-data'],
 subprocess.check_call(['./tools/setup/emoji/build_emoji'],
                       stdout=fp, stderr=fp)
 
+# Build pygment data
+subprocess.check_call(['./tools/setup/build_pygments_data.py'],
+                      stdout=fp, stderr=fp)
+
 # Compile Handlebars templates and minify JavaScript.
 subprocess.check_call(['./tools/minify-js'] +
                       (['--prev-deploy', prev_deploy] if prev_deploy else []),

version.py

@@ -1,2 +1,2 @@
 ZULIP_VERSION = "1.5.1+git"
-PROVISION_VERSION = '4.20'
+PROVISION_VERSION = '4.21'

zproject/settings.py

@@ -827,6 +827,7 @@ def get_secret(key):
             'node_modules/handlebars/dist/handlebars.runtime.js',
             'third/marked/lib/marked.js',
             'generated/emoji/emoji_codes.js',
+            'generated/pygments_data.js',
             'templates/compiled.js',
             'js/feature_flags.js',
             'js/loading.js',