fix: get Studio working with FastAPI

Procfile

@@ -1 +1 @@
-web: gunicorn --worker-class eventlet -w 1 g2p.app:app --no-sendfile
+web: gunicorn --worker-class eventlet -w 1 g2p.app:APP --no-sendfile

README.md

@@ -200,9 +200,8 @@ Gen DB: this is the part of the textual database that is generated when running
 
 Compiled DB: this contains the same info as Text DB + Gen DB, but in a format optimized for fast reading by the machine. This is what any program using `g2p` reads: `g2p convert`, `readalongs align`, `convertextract`, and also `g2p generate-mapping`. It consists of these files:
 * g2p/mappings/langs/langs.json.gz
-* g2p/mappings/langs/network.pkl
+* g2p/mappings/langs/network.json.gz
 * g2p/static/languages-network.json
-* g2p/static/swagger.json
 
 So, when you write a new g2p mapping for a language, say `lll`, and you want to be able to convert text from `lll` to `eng-ipa` or `eng-arpabet`, you need to do the following:
 1. Write the mapping from `lll` to `lll-ipa` in g2p/mappings/langs/lll/. You've just updated Text DB.
@@ -214,18 +213,19 @@ Once you have the Compiled DB, it is then possible to use the `g2p convert` comm
 
 ## Studio
 
-You can also run the `g2p Studio` which is a web interface for creating custom lookup tables to be used with g2p. To run the `g2p Studio` either visit https://g2p-studio.herokuapp.com/ or run it locally with:
-
-    uvicorn g2p.app:app --reload --port 5000
+You can also run the `g2p Studio` which is a web interface for
+creating custom lookup tables to be used with g2p. To run the `g2p
+Studio` either visit https://g2p-studio.herokuapp.com/ or run it
+locally with `run_studio.py`.
 
 ## API for Developers
 
 There is also a REST API available for use in your own applications.
 To launch it from the command-line use `python run_studio.py` or
-`uvicorn g2p.app:app`.  The API documentation will be viewable
+`uvicorn g2p.app:APP`.  The API documentation will be viewable
 (with the ability to use it interactively) at
-http://localhost:5000/docs - an OpenAPI definition is also available
-at http://localhost:5000/static/swagger.json .
+http://localhost:5000/api/v1/docs - an OpenAPI definition is also available
+at http://localhost:5000/api/v1/openapi.json .
 
 ## Maintainers
 

docs/migration-2.md

@@ -100,22 +100,14 @@ been using them and should not be construed as public API documentation.
 
 ## Some CLI commands no longer exist
 
-Several commands for the `g2p` command-line have been removed as they
-were duplicates of other functionality:
+Several commands for the `g2p` command-line have been removed:
 
 - run
 - routes
 - shell
 
-To run the `g2p` API server, you can use:
-
-    flask --app g2p.app run
-
-Likewise, for `routes` and `shell`, you can use:
-
-    flask --app g2p.app routes
-    flask --app g2p.app shell
-
-To run G2P Studio, use:
+To run the `g2p` API server and G2P Studio, you can use:
 
     python run_studio.py
+
+It does not seem that any equivalents of `routes` or `shell` exist.

g2p/app.py

@@ -3,12 +3,12 @@
 
 You can run the app (and API) for development purposes on any platform with:
     pip install uvicorn
-    uvicorn g2p.app:app --reload --port 5000
+    uvicorn g2p.app:APP --reload --port 5000
 - The --reload switch will watch for changes under the directory where it's
   running and reload the code whenever it changes.
 
 You can also spin up the app server grade (on Linux, not Windows) with gunicorn:
-    gunicorn -w 4 -k uvicorn.workers.UvicornWorker g2p.app:append --port 5000
+    gunicorn -w 4 -k uvicorn.workers.UvicornWorker g2p.app:APP --port 5000
 
 Once spun up, the application will be visible at
 http://localhost:5000/ and the API at http://localhost:5000/api/v1/docs
@@ -21,13 +21,13 @@
 from fastapi.responses import HTMLResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
-from fastapi_socketio import SocketManager
-from networkx.algorithms.dag import ancestors, descendants  # type: ignore
+from fastapi_socketio import SocketManager  # type: ignore
+from networkx import shortest_path  # type: ignore
 
 from g2p import make_g2p
 from g2p.api import api as api_v1
 from g2p.log import LOGGER
-from g2p.mappings import Mapping
+from g2p.mappings import Mapping, Rule
 from g2p.mappings.langs import LANGS_NETWORK
 from g2p.mappings.utils import (
     _MappingModelDefinition,
@@ -43,11 +43,18 @@
 
 DEFAULT_N = 10
 
-templates = Jinja2Templates(directory="g2p/templates")
-app = FastAPI()
-socket_manager = SocketManager(app=app)
-app.mount("/api/v1", api_v1)
-app.mount("/static", StaticFiles(directory="g2p/static"), name="static")
+TEMPLATES = Jinja2Templates(directory="g2p/templates")
+APP = FastAPI()
+SOCKET_MANAGER = SocketManager(
+    app=APP,
+    # This next argument is very important
+    # and requires FastApi>=0.109.0
+    # (and thus Starlette>=0.33.0).
+    # See https://github.com/encode/starlette/discussions/2413
+    socketio_path="/ws/socket.io",
+)
+APP.mount("/api/v1", api_v1)
+APP.mount("/static", StaticFiles(directory="g2p/static"), name="static")
 
 
 def shade_colour(colour, percent, r=0, g=0, b=0):
@@ -138,41 +145,26 @@ def return_echart_data(tg: Union[CompositeTransductionGraph, TransductionGraph])
     return nodes, edges
 
 
-def return_empty_mappings(n=DEFAULT_N):
-    """Return 'n' * empty mappings"""
-    y = 0
-    mappings = []
-    while y < n:
-        mappings.append(
-            {"in": "", "out": "", "context_before": "", "context_after": ""}
-        )
-        y += 1
-    return mappings
-
-
-def return_descendant_nodes(node: str):
-    """Return possible outputs for a given input"""
-    return [x for x in descendants(LANGS_NETWORK, node)]
-
-
-@app.get("/", response_class=HTMLResponse)
+@APP.get("/", response_class=HTMLResponse)
 def home(request: Request):
     """Return homepage of g2p studio"""
-    return templates.TemplateResponse("index.html", {"request": request})
+    return TEMPLATES.TemplateResponse("index.html", {"request": request})
 
 
-@app.sio.on("conversion event", namespace="/convert")  # type: ignore
+@APP.sio.on("conversion event", namespace="/convert")  # type: ignore
 async def convert(sid, message):
     """Convert input text and return output"""
     transducers = []
+    LOGGER.debug("/convert: %s", message)
     for mapping in message["data"]["mappings"]:
         mapping_args = {**mapping["kwargs"]}
         mapping_args["abbreviations"] = flatten_abbreviations_format(
             mapping["abbreviations"]
         )
         if mapping_args["type"] == "lexicon":
-            lexicon = Mapping.find_mapping(mapping_args["in_lang"],
-                                           mapping_args["out_lang"])
+            lexicon = Mapping.find_mapping(
+                mapping_args["in_lang"], mapping_args["out_lang"]
+            )
             mapping_args["alignments"] = lexicon.alignments
         else:
             mapping_args["rules"] = mapping["rules"]
@@ -188,13 +180,18 @@ async def convert(sid, message):
                 e,
             )
     if len(transducers) == 0:
-        emit("conversion response", {"output_string": message["data"]["input_string"]})
+        await APP.sio.emit(
+            "conversion response",
+            {"output_string": message["data"]["input_string"]},
+            sid,
+            namespace="/convert",
+        )
         return
     transducer = CompositeTransducer(transducers)
     if message["data"]["index"]:
         tg = transducer(message["data"]["input_string"])
         data, links = return_echart_data(tg)
-        await app.sio.emit(  # type: ignore
+        await APP.sio.emit(  # type: ignore
             "conversion response",
             {
                 "output_string": tg.output_string,
@@ -206,19 +203,25 @@ async def convert(sid, message):
         )
     else:
         output_string = transducer(message["data"]["input_string"]).output_string
-        await app.sio.emit(  # type: ignore
+        await APP.sio.emit(  # type: ignore
             "conversion response",
             {"output_string": output_string},
             sid,
             namespace="/convert",
         )
 
 
-@app.sio.on("table event", namespace="/table")  # type: ignore
+@APP.sio.on("table event", namespace="/table")  # type: ignore
 async def change_table(sid, message):
     """Change the lookup table"""
+    LOGGER.debug("/table: %s", message)
     if "in_lang" not in message or "out_lang" not in message:
-        emit("table response", [])
+        await APP.sio.emit(
+            "table response",
+            [],
+            sid,
+            namespace="/table",
+        )
     elif message["in_lang"] == "custom" or message["out_lang"] == "custom":
         # These are only used to generate JSON to send to the client,
         # so it's safe to create a list of references to the same thing.
@@ -234,9 +237,13 @@ async def change_table(sid, message):
             out_lang="custom",
             type="mapping",
             norm_form="NFC",
+            # Put something here to silence a warning
+            rules=[Rule(rule_input="a", rule_output="a")],
         ).model_dump()
+        kwargs["rules"] = []
+        # Remove the bogus rule we used to silence the validator
         kwargs["include"] = False
-        emit(
+        await APP.sio.emit(
             "table response",
             [
                 {
@@ -245,41 +252,41 @@ async def change_table(sid, message):
                     "kwargs": kwargs,
                 }
             ],
+            sid,
+            namespace="/table",
         )
     else:
-        transducer = make_g2p(message["in_lang"], message["out_lang"])
-    if isinstance(transducer, Transducer):
-        mappings = [transducer.mapping]
-    elif isinstance(transducer, CompositeTransducer):
-        mappings = [x.mapping for x in transducer._transducers]
-    else:
-        pass
-    await app.sio.emit(  # type: ignore
-        "table response",
-        [
-            {
-                "mappings": x.plain_mapping(),
-                "abbs": expand_abbreviations_format(x.abbreviations),
-                "kwargs": x.kwargs,
-            }
-            for x in mappings
-        ],
-        sid,
-        namespace="/table",
-    )
+        path = shortest_path(LANGS_NETWORK, message["in_lang"], message["out_lang"])
+        mappings: List[Mapping] = []
+        for lang1, lang2 in zip(path[:-1], path[1:]):
+            transducer = make_g2p(lang1, lang2, tokenize=False)
+            mappings.append(transducer.mapping)
+        await APP.sio.emit(
+            "table response",
+            [
+                {
+                    "mappings": x.plain_mapping(),
+                    "abbs": expand_abbreviations_format(x.abbreviations),
+                    "kwargs": x.model_dump(exclude=["alignments"]),
+                }
+                for x in mappings
+            ],
+            sid,
+            namespace="/table",
+        )
 
 
-@app.sio.on("connect", namespace="/connect")  # type: ignore
+@APP.sio.on("connect", namespace="/connect")  # type: ignore
 async def test_connect(sid, message):
     """Let client know disconnected"""
-    await app.sio.emit(  # type: ignore
+    await APP.sio.emit(  # type: ignore
         "connection response", {"data": "Connected"}, sid, namespace="/connect"
     )
 
 
-@app.sio.on("disconnect", namespace="/connect")  # type: ignore
+@APP.sio.on("disconnect", namespace="/connect")  # type: ignore
 async def test_disconnect(sid):
     """Let client know disconnected"""
-    await app.sio.emit(  # type: ignore
+    await APP.sio.emit(  # type: ignore
         "connection response", {"data": "Disconnected"}, sid, namespace="/connect"
     )

g2p/cli.py

@@ -648,7 +648,6 @@ def doctor(mapping, list_all, list_ipa):
 def update(in_dir, out_dir):
     """Update cached language files."""
     # Defer expensive imports
-    from g2p.api import update_docs
     from g2p.mappings.langs import reload_db
     from g2p.mappings.langs.utils import cache_langs, network_to_echart
 
@@ -665,7 +664,6 @@ def update(in_dir, out_dir):
     if in_dir == LANGS_DIR and out_dir is None:
         # We only update the documentation when updating using the default directories
         reload_db()
-        update_docs()  # updates g2p/static/swagger.json
         network_to_echart(
             outfile=os.path.join(os.path.dirname(static_file), "languages-network.json")
         )  # updates g2p/status/languages-network.json

g2p/static/custom.js

@@ -1,5 +1,3 @@
-localStorage.debug = '*';
-
 var TABLES = []
 var ABBS = []
 
@@ -428,10 +426,10 @@ var setKwargs = function(index, kwargs) {
     convert()
 }
 
-var socket = ({path: "/ws/socket.io"});
-var conversionSocket = io('/convert', {path: "/ws/socket.io"});
-var connectionSocket = io('/connect', {path: "/ws/socket.io"});
-var tableSocket = io('/table', {path: "/ws/socket.io"});
+var magicalMysteryOptions = { path: '/ws/socket.io', transports: ['websocket', 'polling', 'flashsocket']};
+var conversionSocket = io('/convert', magicalMysteryOptions)
+var connectionSocket = io('/connect', magicalMysteryOptions)
+var tableSocket = io('/table', magicalMysteryOptions)
 
 var trackIndex = function() {
     return $('#animated-radio').is(":checked")
@@ -644,7 +642,7 @@ $(document).ready(function() {
                 changeTable()
             },
             error: function(xhr, ajaxOptions, thrownError) {
-                if (xhr.status == 404) {
+                if (xhr.status == 404 || xhr.status == 422) {
                     $('#input-langselect option[value=custom]').attr('selected', 'selected');
                     $("#output-langselect").empty();
                     $("#output-langselect").append("<option value='custom' selected>Custom</option>");

g2p/templates/index.html

@@ -163,10 +163,9 @@ <h4>Custom Abbreviations</h4>
 
 <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/handsontable.full.min.js"></script>
 
-<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.6.2/socket.io.min.js"
-	integrity="sha512-mUWPbwx6PZatai4i3+gevCw6LeuQxcMgnJs/DZij22MZ4JMpqKaEvMq0N9TTArSWEb5sdQ9xH68HMUc30eNPEA=="
-	crossorigin="anonymous"
-	referrerpolicy="no-referrer">
+<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.7.5/socket.io.js"
+        integrity="sha512-luMnTJZ7oEchNDZAtQhgjomP1eZefnl82ruTH/3Oj/Yu5qYtwL7+dVRccACS/Snp1lFXq188XFipHKYE75IaQQ=="
+        crossorigin="anonymous" referrerpolicy="no-referrer"></script>
 </script>
 
 <script src="https://cdnjs.cloudflare.com/ajax/libs/echarts/4.2.1/echarts.min.js"