Merge pull request hylang#2444 from scauligi/tagreads_spaced

Reader macros always take a full identifier

NEWS.rst

@@ -7,6 +7,14 @@ Removals
 ------------------------------
 * Python 3.7 is no longer supported.
 
+Breaking Changes
+------------------------------
+* Reader macros now always read a full identifier after the initial `#`,
+  allowing for reader macros that start with characters such as `*`, `^`, `_`.
+  Forms like `#*word` will attempt to dispatch a macro named `*word`;
+  to unpack a symbol named `word`, write `#* word` (note the space).
+* Reader macro names are no longer mangled.
+
 Bug Fixes
 ------------------------------
 * Fixed an installation failure in some situations when version lookup

hy/core/macros.hy

@@ -60,9 +60,8 @@
   and its base class :py:class:`Reader <hy.reader.reader.Reader>` for details
   regarding the available processing methods.
 
-  Reader macro names can be any symbol that does not start with a ``^`` and are
-  callable by prefixing the name with a ``#``. i.e. ``(defreader upper ...)`` is
-  called with ``#upper``.
+  Reader macro names can be any valid identifier and are callable by prefixing
+  the name with a ``#``. i.e. ``(defreader upper ...)`` is called with ``#upper``.
 
   Examples:
 
@@ -99,22 +98,19 @@
   (when (not (isinstance key hy.models.Symbol))
     (raise (ValueError f"expected a name, but got {key}")))
 
-  (when (.startswith key "^")
-    (raise (ValueError "reader macro cannot start with a ^")))
-
   (if (and body (isinstance (get body 0) hy.models.String))
       (setv [docstr #* body] body)
       (setv docstr None))
 
-  (setv dispatch-key (hy.mangle (+ "#" (str key))))
+  (setv dispatch-key (str key))
   `(do (eval-and-compile
          (hy.macros.reader-macro
            ~dispatch-key
            (fn [&reader &key]
              ~@(if docstr [docstr] [])
              ~@body)))
        (eval-when-compile
-         (setv (get hy.&reader.reader-table ~dispatch-key)
+         (setv (get hy.&reader.reader-macros ~dispatch-key)
                (get _hy_reader_macros ~dispatch-key)))))
 
 
@@ -126,13 +122,16 @@
 
    Use ``(help foo)`` instead for help with runtime objects."
    (setv symbol (str symbol))
-   (setv mangled (hy.mangle symbol))
+   (setv namespace
+     (if (= (cut symbol 1) "#")
+       (do (setv symbol (cut symbol 1 None))
+         '_hy_reader_macros)
+       (do (setv symbol (hy.mangle symbol))
+           '_hy_macros)))
    (setv builtins (hy.gensym "builtins"))
    `(do (import builtins :as ~builtins)
-        (help (or (.get _hy_macros ~mangled)
-                  (.get _hy_reader_macros ~mangled)
-                  (.get (. ~builtins _hy_macros) ~mangled)
-                  (.get (. ~builtins _hy_reader_macros) ~mangled)
+        (help (or (.get ~namespace ~symbol)
+                  (.get (. ~builtins ~namespace) ~symbol)
                   (raise (NameError f"macro {~symbol !r} is not defined"))))))
 
 

hy/core/result_macros.py

@@ -1800,7 +1800,7 @@ def compile_require(compiler, expr, root, entries):
             reader_assignments = (
                 "ALL"
                 if readers == Symbol("*")
-                else ["#" + reader for reader in readers[0]]
+                else [str(reader) for reader in readers[0]]
             )
             if require_reader(module_name, compiler.module, reader_assignments):
                 ret += compiler.compile(

hy/macros.py

@@ -185,7 +185,7 @@ def require_reader(source_module, target_module, assignments):
     target_macros = target_namespace.setdefault("_hy_reader_macros", {})
 
     assignments = (
-        source_macros.keys() if assignments == "ALL" else map(mangle, assignments)
+        source_macros.keys() if assignments == "ALL" else assignments
     )
 
     for name in assignments:
@@ -200,12 +200,12 @@ def require_reader(source_module, target_module, assignments):
 def enable_readers(module, reader, names):
     _, namespace = derive_target_module(module, inspect.stack()[1][0])
     names = (
-        namespace["_hy_reader_macros"].keys() if names == "ALL" else map(mangle, names)
+        namespace["_hy_reader_macros"].keys() if names == "ALL" else names
     )
     for name in names:
         if name not in namespace["_hy_reader_macros"]:
             raise NameError(f"reader {name} is not defined")
-        reader.reader_table[name] = namespace["_hy_reader_macros"][name]
+        reader.reader_macros[name] = namespace["_hy_reader_macros"][name]
 
 
 def require(source_module, target_module, assignments, prefix=""):

hy/reader/hy_reader.py

@@ -116,6 +116,17 @@ class HyReader(Reader):
 
     NON_IDENT = set("()[]{};\"'`~")
 
+    def __init__(self):
+        super().__init__()
+
+        # move any reader macros declared using
+        # `reader_for("#...")` to the macro table
+        self.reader_macros = {}
+        for tag in list(self.reader_table.keys()):
+            if tag[0] == '#' and tag[1:]:
+                self.reader_macros[tag[1:]] = self.reader_table.pop(tag)
+
+
     def fill_pos(self, model, start):
         """Attach line/col information to a model.
 
@@ -342,37 +353,21 @@ def tag_dispatch(self, key):
 
         Reads a full identifier after the `#` and calls the corresponding handler
         (this allows, e.g., `#reads-multiple-forms foo bar baz`).
-
-        Failing that, reads a single character after the `#` and immediately
-        calls the corresponding handler (this allows, e.g., `#*args` to parse
-        as `#*` followed by `args`).
         """
 
-        if not self.peekc():
+        if not self.peekc().strip():
             raise PrematureEndOfInput.from_reader(
                 "Premature end of input while attempting dispatch", self
             )
 
-        if self.peek_and_getc("^"):
-            typ = self.parse_one_form()
-            target = self.parse_one_form()
-            return mkexpr("annotate", target, typ)
-
-        tag = None
         # try dispatching tagged ident
-        ident = self.read_ident(just_peeking=True)
-        if ident and mangle(key + ident) in self.reader_table:
-            self.getn(len(ident))
-            tag = mangle(key + ident)
-        # failing that, dispatch tag + single character
-        elif key + self.peekc() in self.reader_table:
-            tag = key + self.getc()
-        if tag:
-            tree = self.dispatch(tag)
+        ident = self.read_ident() or self.getc()
+        if ident in self.reader_macros:
+            tree = self.reader_macros[ident](self, ident)
             return as_model(tree) if tree is not None else None
 
         raise LexException.from_reader(
-            f"reader macro '{key + self.read_ident()}' is not defined", self
+            f"reader macro '{key + ident}' is not defined", self
         )
 
     @reader_for("#_")
@@ -382,18 +377,21 @@ def discard(self, _):
         return None
 
     @reader_for("#*")
-    def hash_star(self, _):
+    @reader_for("#**")
+    def hash_star(self, stars):
         """Unpacking forms `#*` and `#**`, corresponding to `*` and `**` in Python."""
-        num_stars = 1
-        while self.peek_and_getc("*"):
-            num_stars += 1
-        if num_stars > 2:
-            raise LexException.from_reader("too many stars", self)
         return mkexpr(
-            "unpack-" + ("iterable", "mapping")[num_stars - 1],
+            "unpack-" + {"*": "iterable", "**": "mapping"}[stars],
             self.parse_one_form(),
         )
 
+    @reader_for("#^")
+    def annotate(self, _):
+        """Annotate a symbol, usually with a type."""
+        typ = self.parse_one_form()
+        target = self.parse_one_form()
+        return mkexpr("annotate", target, typ)
+
     ###
     # Strings
     # (these are more complicated because f-strings

tests/native_tests/comprehensions.hy

@@ -263,7 +263,7 @@
   (assert (= out "x1-x2-y1y2-z1-z2-")))
 
 
-(defmacro eval-isolated [#*body]
+(defmacro eval-isolated [#* body]
   `(hy.eval '(do ~@body) :module "<test>" :locals {}))
 
 

tests/native_tests/functions.hy

@@ -86,8 +86,8 @@
 
 (defn test-defn-annotations []
 
-  (defn #^int f [#^(get List int) p1 p2 #^str p3 #^str [o1 None] #^int [o2 0]
-           #^str #* rest #^str k1 #^int [k2 0] #^bool #** kwargs])
+  (defn #^ int f [#^ (get List int) p1 p2 #^ str p3 #^ str [o1 None] #^ int [o2 0]
+           #^ str #* rest #^ str k1 #^ int [k2 0] #^ bool #** kwargs])
 
   (assert (is (. f __annotations__ ["return"]) int))
   (for [[k v] (.items (dict

tests/native_tests/let.hy

@@ -505,7 +505,7 @@
                   #(10 20 30)))))
 
 
-(defmacro eval-isolated [#*body]
+(defmacro eval-isolated [#* body]
   `(hy.eval '(do ~@body) :module "<test>" :locals {}))
 
 

tests/native_tests/mangling.hy

@@ -109,14 +109,18 @@
   (assert (= x "aabb")))
 
 
-(defreader rm---x
+(defreader rm---
   (setv form (.parse-one-form &reader))
-  [form form])
+  `(do (+= ~form "a")
+       ~form))
+(defreader rm___
+  (setv form (.parse-one-form &reader))
+  `(do (+= ~form "b")
+       ~form))
 (defn test-reader-macro []
   (setv x "")
-  (assert (= #rm---x (do (+= x "a") 1) [1 1]))
-  (assert (= #rm___x (do (+= x "b") 2) [2 2]))
-  (assert (= x "aabb")))
+  (assert (= #rm--- x "a"))
+  (assert (= #rm___ x "ab")))
 
 
 (defn test-special-form []

tests/native_tests/match.hy

@@ -6,8 +6,8 @@
         hy.errors [HySyntaxError])
 
 (defclass [dataclass] Point []
-    (#^int x)
-    (#^int y))
+    (#^ int x)
+    (#^ int y))
 
 (defn test-pattern-matching []
   (assert (is (match 0
@@ -260,10 +260,10 @@
                     _ _)))
     (assert (= {"a" 1  "c" 3}
                (match y
-                      {"b" b #**e} e)))
+                      {"b" b #** e} e)))
     (assert (= {"a" 1  "c" 3}
                (match y
-                      {"b" b #**a} a)))
+                      {"b" b #** a} a)))
     (assert (= b 2))))
 
 

tests/native_tests/other.hy

@@ -44,8 +44,8 @@
 
 (defn test-variable-annotations []
   (defclass AnnotationContainer []
-    (setv #^int x 1 y 2)
-    (#^bool z))
+    (setv #^ int x 1 y 2)
+    (#^ bool z))
 
   (setv annotations (get-type-hints AnnotationContainer))
   (assert (= (get annotations "x") int))

tests/native_tests/reader_macros.hy

@@ -4,6 +4,7 @@
   types
   contextlib [contextmanager]
   hy.errors [HyMacroExpansionError]
+  hy.reader.exceptions [PrematureEndOfInput]
 
   pytest)
 
@@ -30,8 +31,14 @@
 
 (defn test-reader-macros []
   (assert (= (eval-module #[[(defreader foo '1) #foo]]) 1))
-  (assert (in (hy.mangle "#foo")
+  (assert (in "foo"
     (eval-module #[[(defreader foo '1) _hy_reader_macros]])))
+  (assert (= (eval-module #[[(defreader ^foo '1) #^foo]]) 1))
+
+  (assert (not-in "rm___x"
+    (eval-module
+      #[[(defreader rm---x '1)
+         _hy_reader_macros]])))
 
   ;; Assert reader macros operating exclusively at read time
   (with [module (temp-module "<test>")]
@@ -47,39 +54,38 @@
 (defn test-bad-reader-macro-name []
   (with [(pytest.raises HyMacroExpansionError)]
     (eval-module "(defreader :a-key '1)"))
-
-  (with [(pytest.raises HyMacroExpansionError)]
-    (eval-module "(defreader ^foo '1)")))
+  (with [(pytest.raises PrematureEndOfInput)]
+      (eval-module "# _ 3")))
 
 (defn test-require-readers []
   (with [module (temp-module "<test>")]
-    (setv it (hy.read-many #[[(require tests.resources.tlib :readers [upper])
-                             #upper hello]]))
+    (setv it (hy.read-many #[[(require tests.resources.tlib :readers [upper!])
+                             #upper! hello]]))
     (eval-isolated (next it) module)
     (assert (= (next it) 'HELLO)))
 
   ;; test require :readers & :macros is order independent
-  (for [s ["[qplah] :readers [upper]"
-           ":readers [upper] [qplah]"
-           ":macros [qplah] :readers [upper]"
-           ":readers [upper] :macros [qplah]"]]
+  (for [s ["[qplah] :readers [upper!]"
+           ":readers [upper!] [qplah]"
+           ":macros [qplah] :readers [upper!]"
+           ":readers [upper!] :macros [qplah]"]]
     (assert (=
       (eval-module #[f[
         (require tests.resources.tlib {s})
-        [(qplah 1) #upper "hello"]]f])
+        [(qplah 1) #upper! "hello"]]f])
       [[8 1] "HELLO"])))
 
   ;; test require :readers *
   (assert (=
       (eval-module #[=[
         (require tests.resources.tlib :readers *)
-        [#upper "eVeRy" #lower "ReAdEr"]]=])
+        [#upper! "eVeRy" #lower "ReAdEr"]]=])
       ["EVERY" "reader"]))
 
   ;; test can't redefine :macros or :readers assignment brackets
   (with [(pytest.raises hy.errors.HySyntaxError)]
-    (eval-module #[[(require tests.resources.tlib [taggart] [upper])]]))
+    (eval-module #[[(require tests.resources.tlib [taggart] [upper!])]]))
   (with [(pytest.raises hy.errors.HySyntaxError)]
-    (eval-module #[[(require tests.resources.tlib :readers [taggart] :readers [upper])]]))
+    (eval-module #[[(require tests.resources.tlib :readers [taggart] :readers [upper!])]]))
   (with [(pytest.raises hy.errors.HyRequireError)]
     (eval-module #[[(require tests.resources.tlib :readers [not-a-real-reader])]])))