[Gardening] Cleanup TokenKinds.def (swiftlang#10034)

* [Parse] Refactored internal structure of Tokens.def and documented usage.

Added a level of structure to the macro definitions to allow Swift
keywords to be cleanly accessed separately from SIL and Swift keywords
together. Documented structure and usage.

* [Parse] Made use of new guarantees and abstractions in Tokens.def

Used guarantees about undefining macros after import and new
SWIFT_KEYWORD abstraction to simplify usage of the Token.def
imports.

* Gardening

include/swift/Syntax/TokenKinds.def

@@ -9,59 +9,95 @@
 // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 //
 //===----------------------------------------------------------------------===//
-//
-// This file defines macros used for macro-metaprogramming lexer tokens.
-//
+///
+/// This file defines x-macros used for metaprogramming with lexer tokens.
+///
+/// KEYWORD(kw)
+///   SWIFT_KEYWORD(kw)
+///     DECL_KEYWORD(kw)
+///     STMT_KEYWORD(kw)
+///     EXPR_KEYWORD(kw)
+///     PAT_KEYWORD(kw)
+///   SIL_KEYWORD(kw)
+/// POUND_KEYWORD(kw)
+///   POUND_OBJECT_LITERAL(kw, desc, proto)
+///   POUND_OLD_OBJECT_LITERAL(kw, new_kw, old_arg, new_arg)
+///   POUND_CONFIG(kw)
+/// PUNCTUATOR(name, str)
+///
 //===----------------------------------------------------------------------===//
 
 /// KEYWORD(kw)
-///   Expands for every Swift keyword, such as 'if', 'else', etc.
+///   Expands by default for every Swift keyword and every SIL keyword, such as
+///   'if', 'else', 'sil_global', etc. If you only want to use Swift keywords
+///   see SWIFT_KEYWORD.
 #ifndef KEYWORD
 #define KEYWORD(kw)
 #endif
 
+/// SWIFT_KEYWORD(kw)
+///   Expands for every Swift keyword.
+#ifndef SWIFT_KEYWORD
+#define SWIFT_KEYWORD(kw) KEYWORD(kw)
+#endif
+
 /// DECL_KEYWORD(kw)
 ///   Expands for every Swift keyword that can be used in a declaration.
 #ifndef DECL_KEYWORD
-#define DECL_KEYWORD(kw) KEYWORD(kw)
+#define DECL_KEYWORD(kw) SWIFT_KEYWORD(kw)
 #endif
 
 /// STMT_KEYWORD(kw)
 ///   Expands for every Swift keyword used in statement grammar.
 #ifndef STMT_KEYWORD
-#define STMT_KEYWORD(kw) KEYWORD(kw)
+#define STMT_KEYWORD(kw) SWIFT_KEYWORD(kw)
+#endif
+
+/// EXPR_KEYWORD(kw)
+///   Expands for every Swift keyword used in an expression, such as 'true',
+///   'false', and 'as'
+#ifndef EXPR_KEYWORD
+#define EXPR_KEYWORD(kw) SWIFT_KEYWORD(kw)
+#endif
+
+/// PAT_KEYWORD(kw)
+///   Expands for every Swift keyword used in a pattern, which is currently
+///   limited to '_'
+#ifndef PAT_KEYWORD
+#define PAT_KEYWORD(kw) SWIFT_KEYWORD(kw)
+#endif
+
+/// SIL_KEYWORD(kw)
+///   Expands for every SIL keyword. These are only keywords when parsing SIL.
+#ifndef SIL_KEYWORD
+#define SIL_KEYWORD(kw) KEYWORD(kw)
 #endif
 
 /// POUND_KEYWORD(kw)
-///   Every keyword in the #foo namespace.
+///   Every keyword prefixed with a '#'.
 #ifndef POUND_KEYWORD
 #define POUND_KEYWORD(kw)
 #endif
 
 /// POUND_OBJECT_LITERAL(kw, desc, proto)
-///   Every keyword in the #foo namespace representing an object literal.
+///   Every keyword prefixed with a '#' representing an object literal.
 #ifndef POUND_OBJECT_LITERAL
 #define POUND_OBJECT_LITERAL(kw, desc, proto) POUND_KEYWORD(kw)
 #endif
 
 /// POUND_OLD_OBJECT_LITERAL(kw, new_kw, old_arg, new_arg)
-///   Every keyword in the #foo namespace representing an object literal.
+///   Every keyword prefixed with a '#' representing the obsoleted form of an
+///   object literal.
 #ifndef POUND_OLD_OBJECT_LITERAL
 #define POUND_OLD_OBJECT_LITERAL(kw, new_kw, old_arg, new_arg) POUND_KEYWORD(kw)
 #endif
 
 /// POUND_CONFIG(kw)
-///   Every keyword in the #foo namespace representing a configuration.
+///   Every keyword prefixed with a '#' representing a configuration.
 #ifndef POUND_CONFIG
 #define POUND_CONFIG(kw) POUND_KEYWORD(kw)
 #endif
 
-/// SIL_KEYWORD(kw)
-///   Expands for every SIL keyword. These are only keywords when parsing SIL.
-#ifndef SIL_KEYWORD
-#define SIL_KEYWORD(kw) KEYWORD(kw)
-#endif
-
 /// PUNCTUATOR(name, str)
 ///   Expands for every Swift punctuator.
 ///   \param name  The symbolic name of the punctuator, such as
@@ -128,27 +164,27 @@ STMT_KEYWORD(where)
 STMT_KEYWORD(catch)
 
 // Expression keywords.
-KEYWORD(as)
-KEYWORD(Any)
-KEYWORD(false)
-KEYWORD(is)
-KEYWORD(nil)
-KEYWORD(rethrows)
-KEYWORD(super)
-KEYWORD(self)
-KEYWORD(Self)
-KEYWORD(throw)
-KEYWORD(true)
-KEYWORD(try)
-KEYWORD(throws)
+EXPR_KEYWORD(as)
+EXPR_KEYWORD(Any)
+EXPR_KEYWORD(false)
+EXPR_KEYWORD(is)
+EXPR_KEYWORD(nil)
+EXPR_KEYWORD(rethrows)
+EXPR_KEYWORD(super)
+EXPR_KEYWORD(self)
+EXPR_KEYWORD(Self)
+EXPR_KEYWORD(throw)
+EXPR_KEYWORD(true)
+EXPR_KEYWORD(try)
+EXPR_KEYWORD(throws)
 KEYWORD(__FILE__)
 KEYWORD(__LINE__)
 KEYWORD(__COLUMN__)
 KEYWORD(__FUNCTION__)
 KEYWORD(__DSO_HANDLE__)
 
 // Pattern keywords.
-KEYWORD(_)
+PAT_KEYWORD(_)
 
 // Punctuators.
 PUNCTUATOR(l_paren,       "(")
@@ -189,7 +225,7 @@ PUNCTUATOR(sil_exclamation, "!")    // Only in SIL mode.
 PUNCTUATOR(l_square_lit,  "[#")
 PUNCTUATOR(r_square_lit,  "#]")
 
-// Keywords in the # namespace.  "if" becomes "tok::pound_if".
+// Keywords prefixed with a '#'.  "if" becomes "tok::pound_if".
 POUND_KEYWORD(if)
 POUND_KEYWORD(else)
 POUND_KEYWORD(elseif)
@@ -199,12 +235,11 @@ POUND_KEYWORD(line)
 POUND_KEYWORD(sourceLocation)
 POUND_KEYWORD(selector)
 
-// Keywords in the # namespace that are build configurations.
+// Keywords prefixed with a '#' that are build configurations.
 POUND_CONFIG(available)
 
 
-// Declaratively define object literals, including their
-// corresponding protocols.
+// Object literals and their corresponding protocols.
 POUND_OBJECT_LITERAL(fileLiteral, "file reference", ExpressibleByFileReferenceLiteral)
 POUND_OBJECT_LITERAL(imageLiteral, "image", ExpressibleByImageLiteral)
 POUND_OBJECT_LITERAL(colorLiteral, "color", ExpressibleByColorLiteral)
@@ -220,11 +255,14 @@ POUND_KEYWORD(dsohandle)
 
 
 #undef KEYWORD
+#undef SWIFT_KEYWORD
 #undef DECL_KEYWORD
 #undef STMT_KEYWORD
+#undef EXPR_KEYWORD
+#undef PAT_KEYWORD
 #undef SIL_KEYWORD
-#undef PUNCTUATOR
 #undef POUND_KEYWORD
 #undef POUND_OBJECT_LITERAL
 #undef POUND_OLD_OBJECT_LITERAL
 #undef POUND_CONFIG
+#undef PUNCTUATOR

lib/Parse/Lexer.cpp

@@ -650,7 +650,6 @@ void Lexer::lexHash() {
 
   // Map the character sequence onto
   tok Kind = llvm::StringSwitch<tok>(StringRef(CurPtr, tmpPtr-CurPtr))
-#define KEYWORD(kw)
 #define POUND_KEYWORD(id) \
   .Case(#id, tok::pound_##id)
 #include "swift/Syntax/TokenKinds.def"

tools/SourceKit/lib/SwiftLang/SwiftCompletion.cpp

@@ -590,11 +590,10 @@ getCodeCompletionLiteralKindForUID(UIdent uid) {
 
 static CodeCompletionKeywordKind
 getCodeCompletionKeywordKindForUID(UIdent uid) {
-#define SIL_KEYWORD(kw)
-#define KEYWORD(kw)                                                            \
-  static UIdent Keyword##kw##UID("source.lang.swift.keyword." #kw);            \
-  if (uid == Keyword##kw##UID) {                                               \
-    return CodeCompletionKeywordKind::kw_##kw;                                 \
+#define SWIFT_KEYWORD(kw) \
+  static UIdent Keyword##kw##UID("source.lang.swift.keyword." #kw); \
+  if (uid == Keyword##kw##UID) { \
+    return CodeCompletionKeywordKind::kw_##kw; \
   }
 #include "swift/Syntax/TokenKinds.def"