From 9934ee91c7077646b339040265e342e6fb7f3ae1 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Sun, 8 Oct 2023 21:57:07 -0700 Subject: [PATCH 1/4] Improve perf with finetuned model --- app/course/tasks.py | 12 +- app/llm/generators/concepts.py | 8 +- app/llm/generators/lesson.py | 2 +- app/llm/generators/outline.py | 13 +- app/settings.py | 3 +- app/util.py | 23 +-- poetry.lock | 248 +++++++++++++++++++++------------ pyproject.toml | 2 + 8 files changed, 200 insertions(+), 111 deletions(-) diff --git a/app/course/tasks.py b/app/course/tasks.py index 56676dc..88f527a 100644 --- a/app/course/tasks.py +++ b/app/course/tasks.py @@ -1,11 +1,14 @@ from typing import List +from tenacity import RetryError + from app.course.embeddings import EmbeddingContext from app.course.schemas import ResearchNote from app.llm.exceptions import GenerationError, InvalidRequestError, RateLimitError from app.llm.generators.concepts import generate_concepts from app.llm.generators.outline import generate_outline from app.services.generators.pdf import download_and_parse_pdfs, search_pdfs +from app.settings import settings from app.util import debug_print_trace @@ -13,13 +16,12 @@ async def create_course_concepts(course_name: str, revision: int): """ Set the topic and concepts for a course async. """ - topic = None generated_concepts = None try: - concepts = await generate_concepts(course_name, revision) + concepts = await generate_concepts(course_name, revision, include_examples=settings.INCLUDE_EXAMPLES) if concepts.feasible: generated_concepts = concepts.concepts - except (GenerationError, RateLimitError, InvalidRequestError) as e: + except (GenerationError, RateLimitError, InvalidRequestError, RetryError) as e: debug_print_trace() print(f"Error generating concepts for {course_name}: {e}") @@ -32,13 +34,13 @@ async def create_course_outline( outline_list = None queries = None try: - response = generate_outline(course_name, concepts, revision, item_count=outline_items) + response = generate_outline(course_name, concepts, revision, item_count=outline_items, include_examples=settings.INCLUDE_EXAMPLES) # Stream outline as it generates async for outline_data in response: outline_list = outline_data.outline queries = outline_data.queries - except (GenerationError, RateLimitError, InvalidRequestError) as e: + except (GenerationError, RateLimitError, InvalidRequestError, RetryError) as e: debug_print_trace() print(f"Error generating outline for {course_name}") diff --git a/app/llm/generators/concepts.py b/app/llm/generators/concepts.py index 5b60b4a..19cd8b7 100644 --- a/app/llm/generators/concepts.py +++ b/app/llm/generators/concepts.py @@ -4,6 +4,7 @@ from json import JSONDecodeError from typing import List +import ftfy from pydantic import BaseModel from tenacity import stop_after_attempt, wait_fixed, before, after, retry, retry_if_exception_type import threading @@ -52,14 +53,14 @@ def after_retry_callback(retry_state): @retry( retry=retry_if_exception_type(GenerationError), - stop=stop_after_attempt(2), + stop=stop_after_attempt(5), wait=wait_fixed(2), before_sleep=before_retry_callback, after=after_retry_callback, reraise=True, ) -async def generate_concepts(topic: str, revision: int) -> CourseGeneratedConcepts: - prompt = concept_prompt(topic) +async def generate_concepts(topic: str, revision: int, include_examples: bool = True) -> CourseGeneratedConcepts: + prompt = concept_prompt(topic, include_examples=include_examples) text = "" # If we should cache the prompt - skip cache if we're retrying should_cache = not getattr(local_data, "is_retry", False) @@ -68,6 +69,7 @@ async def generate_concepts(topic: str, revision: int) -> CourseGeneratedConcept text += chunk try: text = extract_only_json_dict(text) + text = str(ftfy.fix_text(text)) data = json.loads(text.strip()) concepts = data["concepts"] feasible = data["feasible"] diff --git a/app/llm/generators/lesson.py b/app/llm/generators/lesson.py index 3290c89..403e3ca 100644 --- a/app/llm/generators/lesson.py +++ b/app/llm/generators/lesson.py @@ -12,7 +12,7 @@ lesson_settings = GenerationSettings( temperature=0.4, max_tokens=6000, - timeout=240, + timeout=480, stop_tokens=None, prompt_type="lesson", ) diff --git a/app/llm/generators/outline.py b/app/llm/generators/outline.py index 2cd903e..8914016 100644 --- a/app/llm/generators/outline.py +++ b/app/llm/generators/outline.py @@ -6,6 +6,7 @@ from json import JSONDecodeError from typing import AsyncGenerator, List +import ftfy from pydantic import BaseModel, parse_obj_as from tenacity import retry_if_exception_type, stop_after_attempt, retry, wait_fixed @@ -24,6 +25,8 @@ model=settings.LLM_INSTRUCT_TYPE, ) +# This can get better results from a finetuned model, forces a certain outline format +prompt_start_hint = '\n{"outline": ["1. ' class GeneratedOutlineData(BaseModel): outline: List[str] @@ -43,6 +46,8 @@ def outline_prompt(topic: str, concepts: List[str], item_count: int = settings.S item_count=item_count, include_examples=include_examples, ) + if settings.FINETUNED: + prompt += prompt_start_hint return prompt @@ -79,7 +84,7 @@ def after_retry_callback(retry_state): @retry( retry=retry_if_exception_type(GenerationError), - stop=stop_after_attempt(2), + stop=stop_after_attempt(5), wait=wait_fixed(2), before_sleep=before_retry_callback, after=after_retry_callback, @@ -91,11 +96,14 @@ async def generate_outline( revision: int, update_after_chars: int = 50, item_count: int = 10, + include_examples: bool = True ) -> AsyncGenerator[GeneratedOutlineData, None]: # Sort concepts alphabetically so that the prompt is the same every time concepts = sorted(concepts) - prompt = outline_prompt(topic, concepts, item_count=item_count) + prompt = outline_prompt(topic, concepts, item_count=item_count, include_examples=include_examples) text = "" + if settings.FINETUNED: + text = prompt_start_hint # Do not hit cache on retries should_cache = not getattr(local_data, "is_retry", False) response = generate_response(prompt, outline_settings, cache=should_cache, revision=revision) @@ -114,6 +122,7 @@ async def generate_outline( try: # Strip out text before/after the json. Sometimes the LLM will include something before the json input. text = extract_only_json_dict(text) + text = str(ftfy.fix_text(text)) data = json.loads(text.strip()) except JSONDecodeError as e: raise GenerationError(e) diff --git a/app/settings.py b/app/settings.py index 110184f..4d5cd7e 100644 --- a/app/settings.py +++ b/app/settings.py @@ -19,10 +19,11 @@ class Settings(BaseSettings): # Content SECTIONS_PER_LESSON: int = 30 # Lower this to make books shorter + MAX_DOWNLOAD_SIZE: int = 6 * 1024 * 1024 # Max pdf size to download, 6 MB + FINETUNED: bool = False # If we're using a finetuned textbook gen model INCLUDE_EXAMPLES: bool = ( True # Include examples in prompts, False with custom model ) - MAX_DOWNLOAD_SIZE: int = 6 * 1024 * 1024 # Max pdf size to download, 6 MB # LLM LLM_TYPES = { diff --git a/app/util.py b/app/util.py index 8cef527..ed59995 100644 --- a/app/util.py +++ b/app/util.py @@ -1,5 +1,6 @@ import traceback from enum import Enum +import regex import ftfy @@ -11,17 +12,23 @@ def __str__(self): return str(self.value) -def extract_only_json_list(text: str) -> str: - text = text.split("[", 1)[1] - text = text.rsplit("]", 1)[0] - text = "[" + text + "]" +def extract_only_json_dict(text: str) -> str: + # Extract the first top-level JSON object + pattern = r'({(?:[^{}]|(?R))*})' + match = regex.search(pattern, text, regex.DOTALL) + if match: + return match.group(0) + return text -def extract_only_json_dict(text: str) -> str: - text = text.split("{", 1)[1] - text = text.rsplit("}", 1)[0] - text = "{" + text + "}" +def extract_only_json_list(text: str) -> str: + # Extract the first top-level JSON object + pattern = r'(\[(?:[^\[\]]|(?R))*\])' + match = regex.search(pattern, text, regex.DOTALL) + if match: + return match.group(0) + return text diff --git a/poetry.lock b/poetry.lock index 9e47d43..6cda223 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1068,6 +1068,72 @@ files = [ docs = ["Sphinx", "docutils (<0.18)"] test = ["objgraph", "psutil"] +[[package]] +name = "grpcio" +version = "1.59.0" +description = "HTTP/2-based RPC framework" +optional = false +python-versions = ">=3.7" +files = [ + {file = "grpcio-1.59.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:225e5fa61c35eeaebb4e7491cd2d768cd8eb6ed00f2664fa83a58f29418b39fd"}, + {file = "grpcio-1.59.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:b95ec8ecc4f703f5caaa8d96e93e40c7f589bad299a2617bdb8becbcce525539"}, + {file = "grpcio-1.59.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:1a839ba86764cc48226f50b924216000c79779c563a301586a107bda9cbe9dcf"}, + {file = "grpcio-1.59.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6cfe44a5d7c7d5f1017a7da1c8160304091ca5dc64a0f85bca0d63008c3137a"}, + {file = "grpcio-1.59.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0fcf53df684fcc0154b1e61f6b4a8c4cf5f49d98a63511e3f30966feff39cd0"}, + {file = "grpcio-1.59.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa66cac32861500f280bb60fe7d5b3e22d68c51e18e65367e38f8669b78cea3b"}, + {file = "grpcio-1.59.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8cd2d38c2d52f607d75a74143113174c36d8a416d9472415eab834f837580cf7"}, + {file = "grpcio-1.59.0-cp310-cp310-win32.whl", hash = "sha256:228b91ce454876d7eed74041aff24a8f04c0306b7250a2da99d35dd25e2a1211"}, + {file = "grpcio-1.59.0-cp310-cp310-win_amd64.whl", hash = "sha256:ca87ee6183421b7cea3544190061f6c1c3dfc959e0b57a5286b108511fd34ff4"}, + {file = "grpcio-1.59.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:c173a87d622ea074ce79be33b952f0b424fa92182063c3bda8625c11d3585d09"}, + {file = "grpcio-1.59.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:ec78aebb9b6771d6a1de7b6ca2f779a2f6113b9108d486e904bde323d51f5589"}, + {file = "grpcio-1.59.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:0b84445fa94d59e6806c10266b977f92fa997db3585f125d6b751af02ff8b9fe"}, + {file = "grpcio-1.59.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c251d22de8f9f5cca9ee47e4bade7c5c853e6e40743f47f5cc02288ee7a87252"}, + {file = "grpcio-1.59.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:956f0b7cb465a65de1bd90d5a7475b4dc55089b25042fe0f6c870707e9aabb1d"}, + {file = "grpcio-1.59.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:38da5310ef84e16d638ad89550b5b9424df508fd5c7b968b90eb9629ca9be4b9"}, + {file = "grpcio-1.59.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:63982150a7d598281fa1d7ffead6096e543ff8be189d3235dd2b5604f2c553e5"}, + {file = "grpcio-1.59.0-cp311-cp311-win32.whl", hash = "sha256:50eff97397e29eeee5df106ea1afce3ee134d567aa2c8e04fabab05c79d791a7"}, + {file = "grpcio-1.59.0-cp311-cp311-win_amd64.whl", hash = "sha256:15f03bd714f987d48ae57fe092cf81960ae36da4e520e729392a59a75cda4f29"}, + {file = "grpcio-1.59.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:f1feb034321ae2f718172d86b8276c03599846dc7bb1792ae370af02718f91c5"}, + {file = "grpcio-1.59.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d09bd2a4e9f5a44d36bb8684f284835c14d30c22d8ec92ce796655af12163588"}, + {file = "grpcio-1.59.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:2f120d27051e4c59db2f267b71b833796770d3ea36ca712befa8c5fff5da6ebd"}, + {file = "grpcio-1.59.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba0ca727a173ee093f49ead932c051af463258b4b493b956a2c099696f38aa66"}, + {file = "grpcio-1.59.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5711c51e204dc52065f4a3327dca46e69636a0b76d3e98c2c28c4ccef9b04c52"}, + {file = "grpcio-1.59.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:d74f7d2d7c242a6af9d4d069552ec3669965b74fed6b92946e0e13b4168374f9"}, + {file = "grpcio-1.59.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3859917de234a0a2a52132489c4425a73669de9c458b01c9a83687f1f31b5b10"}, + {file = "grpcio-1.59.0-cp312-cp312-win32.whl", hash = "sha256:de2599985b7c1b4ce7526e15c969d66b93687571aa008ca749d6235d056b7205"}, + {file = "grpcio-1.59.0-cp312-cp312-win_amd64.whl", hash = "sha256:598f3530231cf10ae03f4ab92d48c3be1fee0c52213a1d5958df1a90957e6a88"}, + {file = "grpcio-1.59.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:b34c7a4c31841a2ea27246a05eed8a80c319bfc0d3e644412ec9ce437105ff6c"}, + {file = "grpcio-1.59.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:c4dfdb49f4997dc664f30116af2d34751b91aa031f8c8ee251ce4dcfc11277b0"}, + {file = "grpcio-1.59.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:61bc72a00ecc2b79d9695220b4d02e8ba53b702b42411397e831c9b0589f08a3"}, + {file = "grpcio-1.59.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f367e4b524cb319e50acbdea57bb63c3b717c5d561974ace0b065a648bb3bad3"}, + {file = "grpcio-1.59.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:849c47ef42424c86af069a9c5e691a765e304079755d5c29eff511263fad9c2a"}, + {file = "grpcio-1.59.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c0488c2b0528e6072010182075615620071371701733c63ab5be49140ed8f7f0"}, + {file = "grpcio-1.59.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:611d9aa0017fa386809bddcb76653a5ab18c264faf4d9ff35cb904d44745f575"}, + {file = "grpcio-1.59.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e5378785dce2b91eb2e5b857ec7602305a3b5cf78311767146464bfa365fc897"}, + {file = "grpcio-1.59.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fe976910de34d21057bcb53b2c5e667843588b48bf11339da2a75f5c4c5b4055"}, + {file = "grpcio-1.59.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:c041a91712bf23b2a910f61e16565a05869e505dc5a5c025d429ca6de5de842c"}, + {file = "grpcio-1.59.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:0ae444221b2c16d8211b55326f8ba173ba8f8c76349bfc1768198ba592b58f74"}, + {file = "grpcio-1.59.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ceb1e68135788c3fce2211de86a7597591f0b9a0d2bb80e8401fd1d915991bac"}, + {file = "grpcio-1.59.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c4b1cc3a9dc1924d2eb26eec8792fedd4b3fcd10111e26c1d551f2e4eda79ce"}, + {file = "grpcio-1.59.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:871371ce0c0055d3db2a86fdebd1e1d647cf21a8912acc30052660297a5a6901"}, + {file = "grpcio-1.59.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:93e9cb546e610829e462147ce724a9cb108e61647a3454500438a6deef610be1"}, + {file = "grpcio-1.59.0-cp38-cp38-win32.whl", hash = "sha256:f21917aa50b40842b51aff2de6ebf9e2f6af3fe0971c31960ad6a3a2b24988f4"}, + {file = "grpcio-1.59.0-cp38-cp38-win_amd64.whl", hash = "sha256:14890da86a0c0e9dc1ea8e90101d7a3e0e7b1e71f4487fab36e2bfd2ecadd13c"}, + {file = "grpcio-1.59.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:34341d9e81a4b669a5f5dca3b2a760b6798e95cdda2b173e65d29d0b16692857"}, + {file = "grpcio-1.59.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:986de4aa75646e963466b386a8c5055c8b23a26a36a6c99052385d6fe8aaf180"}, + {file = "grpcio-1.59.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:aca8a24fef80bef73f83eb8153f5f5a0134d9539b4c436a716256b311dda90a6"}, + {file = "grpcio-1.59.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:936b2e04663660c600d5173bc2cc84e15adbad9c8f71946eb833b0afc205b996"}, + {file = "grpcio-1.59.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc8bf2e7bc725e76c0c11e474634a08c8f24bcf7426c0c6d60c8f9c6e70e4d4a"}, + {file = "grpcio-1.59.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81d86a096ccd24a57fa5772a544c9e566218bc4de49e8c909882dae9d73392df"}, + {file = "grpcio-1.59.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2ea95cd6abbe20138b8df965b4a8674ec312aaef3147c0f46a0bac661f09e8d0"}, + {file = "grpcio-1.59.0-cp39-cp39-win32.whl", hash = "sha256:3b8ff795d35a93d1df6531f31c1502673d1cebeeba93d0f9bd74617381507e3f"}, + {file = "grpcio-1.59.0-cp39-cp39-win_amd64.whl", hash = "sha256:38823bd088c69f59966f594d087d3a929d1ef310506bee9e3648317660d65b81"}, + {file = "grpcio-1.59.0.tar.gz", hash = "sha256:acf70a63cf09dd494000007b798aff88a436e1c03b394995ce450be437b8e54f"}, +] + +[package.extras] +protobuf = ["grpcio-tools (>=1.59.0)"] + [[package]] name = "huggingface-hub" version = "0.17.3" @@ -3278,99 +3344,99 @@ rpds-py = ">=0.7.0" [[package]] name = "regex" -version = "2023.8.8" +version = "2023.10.3" description = "Alternative regular expression module, to replace re." optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "regex-2023.8.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:88900f521c645f784260a8d346e12a1590f79e96403971241e64c3a265c8ecdb"}, - {file = "regex-2023.8.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3611576aff55918af2697410ff0293d6071b7e00f4b09e005d614686ac4cd57c"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8a0ccc8f2698f120e9e5742f4b38dc944c38744d4bdfc427616f3a163dd9de5"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c662a4cbdd6280ee56f841f14620787215a171c4e2d1744c9528bed8f5816c96"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf0633e4a1b667bfe0bb10b5e53fe0d5f34a6243ea2530eb342491f1adf4f739"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:551ad543fa19e94943c5b2cebc54c73353ffff08228ee5f3376bd27b3d5b9800"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54de2619f5ea58474f2ac211ceea6b615af2d7e4306220d4f3fe690c91988a61"}, - {file = "regex-2023.8.8-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ec4b3f0aebbbe2fc0134ee30a791af522a92ad9f164858805a77442d7d18570"}, - {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ae646c35cb9f820491760ac62c25b6d6b496757fda2d51be429e0e7b67ae0ab"}, - {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca339088839582d01654e6f83a637a4b8194d0960477b9769d2ff2cfa0fa36d2"}, - {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:d9b6627408021452dcd0d2cdf8da0534e19d93d070bfa8b6b4176f99711e7f90"}, - {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:bd3366aceedf274f765a3a4bc95d6cd97b130d1dda524d8f25225d14123c01db"}, - {file = "regex-2023.8.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7aed90a72fc3654fba9bc4b7f851571dcc368120432ad68b226bd593f3f6c0b7"}, - {file = "regex-2023.8.8-cp310-cp310-win32.whl", hash = "sha256:80b80b889cb767cc47f31d2b2f3dec2db8126fbcd0cff31b3925b4dc6609dcdb"}, - {file = "regex-2023.8.8-cp310-cp310-win_amd64.whl", hash = "sha256:b82edc98d107cbc7357da7a5a695901b47d6eb0420e587256ba3ad24b80b7d0b"}, - {file = "regex-2023.8.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1e7d84d64c84ad97bf06f3c8cb5e48941f135ace28f450d86af6b6512f1c9a71"}, - {file = "regex-2023.8.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ce0f9fbe7d295f9922c0424a3637b88c6c472b75eafeaff6f910494a1fa719ef"}, - {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06c57e14ac723b04458df5956cfb7e2d9caa6e9d353c0b4c7d5d54fcb1325c46"}, - {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7a9aaa5a1267125eef22cef3b63484c3241aaec6f48949b366d26c7250e0357"}, - {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b7408511fca48a82a119d78a77c2f5eb1b22fe88b0d2450ed0756d194fe7a9a"}, - {file = "regex-2023.8.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14dc6f2d88192a67d708341f3085df6a4f5a0c7b03dec08d763ca2cd86e9f559"}, - {file = "regex-2023.8.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48c640b99213643d141550326f34f0502fedb1798adb3c9eb79650b1ecb2f177"}, - {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0085da0f6c6393428bf0d9c08d8b1874d805bb55e17cb1dfa5ddb7cfb11140bf"}, - {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:964b16dcc10c79a4a2be9f1273fcc2684a9eedb3906439720598029a797b46e6"}, - {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7ce606c14bb195b0e5108544b540e2c5faed6843367e4ab3deb5c6aa5e681208"}, - {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:40f029d73b10fac448c73d6eb33d57b34607f40116e9f6e9f0d32e9229b147d7"}, - {file = "regex-2023.8.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3b8e6ea6be6d64104d8e9afc34c151926f8182f84e7ac290a93925c0db004bfd"}, - {file = "regex-2023.8.8-cp311-cp311-win32.whl", hash = "sha256:942f8b1f3b223638b02df7df79140646c03938d488fbfb771824f3d05fc083a8"}, - {file = "regex-2023.8.8-cp311-cp311-win_amd64.whl", hash = "sha256:51d8ea2a3a1a8fe4f67de21b8b93757005213e8ac3917567872f2865185fa7fb"}, - {file = "regex-2023.8.8-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e951d1a8e9963ea51efd7f150450803e3b95db5939f994ad3d5edac2b6f6e2b4"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:704f63b774218207b8ccc6c47fcef5340741e5d839d11d606f70af93ee78e4d4"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22283c769a7b01c8ac355d5be0715bf6929b6267619505e289f792b01304d898"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:91129ff1bb0619bc1f4ad19485718cc623a2dc433dff95baadbf89405c7f6b57"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de35342190deb7b866ad6ba5cbcccb2d22c0487ee0cbb251efef0843d705f0d4"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b993b6f524d1e274a5062488a43e3f9f8764ee9745ccd8e8193df743dbe5ee61"}, - {file = "regex-2023.8.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3026cbcf11d79095a32d9a13bbc572a458727bd5b1ca332df4a79faecd45281c"}, - {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:293352710172239bf579c90a9864d0df57340b6fd21272345222fb6371bf82b3"}, - {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d909b5a3fff619dc7e48b6b1bedc2f30ec43033ba7af32f936c10839e81b9217"}, - {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3d370ff652323c5307d9c8e4c62efd1956fb08051b0e9210212bc51168b4ff56"}, - {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:b076da1ed19dc37788f6a934c60adf97bd02c7eea461b73730513921a85d4235"}, - {file = "regex-2023.8.8-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e9941a4ada58f6218694f382e43fdd256e97615db9da135e77359da257a7168b"}, - {file = "regex-2023.8.8-cp36-cp36m-win32.whl", hash = "sha256:a8c65c17aed7e15a0c824cdc63a6b104dfc530f6fa8cb6ac51c437af52b481c7"}, - {file = "regex-2023.8.8-cp36-cp36m-win_amd64.whl", hash = "sha256:aadf28046e77a72f30dcc1ab185639e8de7f4104b8cb5c6dfa5d8ed860e57236"}, - {file = "regex-2023.8.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:423adfa872b4908843ac3e7a30f957f5d5282944b81ca0a3b8a7ccbbfaa06103"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ae594c66f4a7e1ea67232a0846649a7c94c188d6c071ac0210c3e86a5f92109"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e51c80c168074faa793685656c38eb7a06cbad7774c8cbc3ea05552d615393d8"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:09b7f4c66aa9d1522b06e31a54f15581c37286237208df1345108fcf4e050c18"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e73e5243af12d9cd6a9d6a45a43570dbe2e5b1cdfc862f5ae2b031e44dd95a8"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:941460db8fe3bd613db52f05259c9336f5a47ccae7d7def44cc277184030a116"}, - {file = "regex-2023.8.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f0ccf3e01afeb412a1a9993049cb160d0352dba635bbca7762b2dc722aa5742a"}, - {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:2e9216e0d2cdce7dbc9be48cb3eacb962740a09b011a116fd7af8c832ab116ca"}, - {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:5cd9cd7170459b9223c5e592ac036e0704bee765706445c353d96f2890e816c8"}, - {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4873ef92e03a4309b3ccd8281454801b291b689f6ad45ef8c3658b6fa761d7ac"}, - {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:239c3c2a339d3b3ddd51c2daef10874410917cd2b998f043c13e2084cb191684"}, - {file = "regex-2023.8.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:1005c60ed7037be0d9dea1f9c53cc42f836188227366370867222bda4c3c6bd7"}, - {file = "regex-2023.8.8-cp37-cp37m-win32.whl", hash = "sha256:e6bd1e9b95bc5614a7a9c9c44fde9539cba1c823b43a9f7bc11266446dd568e3"}, - {file = "regex-2023.8.8-cp37-cp37m-win_amd64.whl", hash = "sha256:9a96edd79661e93327cfeac4edec72a4046e14550a1d22aa0dd2e3ca52aec921"}, - {file = "regex-2023.8.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f2181c20ef18747d5f4a7ea513e09ea03bdd50884a11ce46066bb90fe4213675"}, - {file = "regex-2023.8.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a2ad5add903eb7cdde2b7c64aaca405f3957ab34f16594d2b78d53b8b1a6a7d6"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9233ac249b354c54146e392e8a451e465dd2d967fc773690811d3a8c240ac601"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:920974009fb37b20d32afcdf0227a2e707eb83fe418713f7a8b7de038b870d0b"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2b6c5dfe0929b6c23dde9624483380b170b6e34ed79054ad131b20203a1a63"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96979d753b1dc3b2169003e1854dc67bfc86edf93c01e84757927f810b8c3c93"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ae54a338191e1356253e7883d9d19f8679b6143703086245fb14d1f20196be9"}, - {file = "regex-2023.8.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2162ae2eb8b079622176a81b65d486ba50b888271302190870b8cc488587d280"}, - {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c884d1a59e69e03b93cf0dfee8794c63d7de0ee8f7ffb76e5f75be8131b6400a"}, - {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cf9273e96f3ee2ac89ffcb17627a78f78e7516b08f94dc435844ae72576a276e"}, - {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:83215147121e15d5f3a45d99abeed9cf1fe16869d5c233b08c56cdf75f43a504"}, - {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3f7454aa427b8ab9101f3787eb178057c5250478e39b99540cfc2b889c7d0586"}, - {file = "regex-2023.8.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0640913d2c1044d97e30d7c41728195fc37e54d190c5385eacb52115127b882"}, - {file = "regex-2023.8.8-cp38-cp38-win32.whl", hash = "sha256:0c59122ceccb905a941fb23b087b8eafc5290bf983ebcb14d2301febcbe199c7"}, - {file = "regex-2023.8.8-cp38-cp38-win_amd64.whl", hash = "sha256:c12f6f67495ea05c3d542d119d270007090bad5b843f642d418eb601ec0fa7be"}, - {file = "regex-2023.8.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:82cd0a69cd28f6cc3789cc6adeb1027f79526b1ab50b1f6062bbc3a0ccb2dbc3"}, - {file = "regex-2023.8.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bb34d1605f96a245fc39790a117ac1bac8de84ab7691637b26ab2c5efb8f228c"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:987b9ac04d0b38ef4f89fbc035e84a7efad9cdd5f1e29024f9289182c8d99e09"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9dd6082f4e2aec9b6a0927202c85bc1b09dcab113f97265127c1dc20e2e32495"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7eb95fe8222932c10d4436e7a6f7c99991e3fdd9f36c949eff16a69246dee2dc"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7098c524ba9f20717a56a8d551d2ed491ea89cbf37e540759ed3b776a4f8d6eb"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b694430b3f00eb02c594ff5a16db30e054c1b9589a043fe9174584c6efa8033"}, - {file = "regex-2023.8.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b2aeab3895d778155054abea5238d0eb9a72e9242bd4b43f42fd911ef9a13470"}, - {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:988631b9d78b546e284478c2ec15c8a85960e262e247b35ca5eaf7ee22f6050a"}, - {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:67ecd894e56a0c6108ec5ab1d8fa8418ec0cff45844a855966b875d1039a2e34"}, - {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:14898830f0a0eb67cae2bbbc787c1a7d6e34ecc06fbd39d3af5fe29a4468e2c9"}, - {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f2200e00b62568cfd920127782c61bc1c546062a879cdc741cfcc6976668dfcf"}, - {file = "regex-2023.8.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9691a549c19c22d26a4f3b948071e93517bdf86e41b81d8c6ac8a964bb71e5a6"}, - {file = "regex-2023.8.8-cp39-cp39-win32.whl", hash = "sha256:6ab2ed84bf0137927846b37e882745a827458689eb969028af8032b1b3dac78e"}, - {file = "regex-2023.8.8-cp39-cp39-win_amd64.whl", hash = "sha256:5543c055d8ec7801901e1193a51570643d6a6ab8751b1f7dd9af71af467538bb"}, - {file = "regex-2023.8.8.tar.gz", hash = "sha256:fcbdc5f2b0f1cd0f6a56cdb46fe41d2cce1e644e3b68832f3eeebc5fb0f7712e"}, + {file = "regex-2023.10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4c34d4f73ea738223a094d8e0ffd6d2c1a1b4c175da34d6b0de3d8d69bee6bcc"}, + {file = "regex-2023.10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a8f4e49fc3ce020f65411432183e6775f24e02dff617281094ba6ab079ef0915"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cd1bccf99d3ef1ab6ba835308ad85be040e6a11b0977ef7ea8c8005f01a3c29"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:81dce2ddc9f6e8f543d94b05d56e70d03a0774d32f6cca53e978dc01e4fc75b8"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c6b4d23c04831e3ab61717a707a5d763b300213db49ca680edf8bf13ab5d91b"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c15ad0aee158a15e17e0495e1e18741573d04eb6da06d8b84af726cfc1ed02ee"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6239d4e2e0b52c8bd38c51b760cd870069f0bdf99700a62cd509d7a031749a55"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4a8bf76e3182797c6b1afa5b822d1d5802ff30284abe4599e1247be4fd6b03be"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9c727bbcf0065cbb20f39d2b4f932f8fa1631c3e01fcedc979bd4f51fe051c5"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3ccf2716add72f80714b9a63899b67fa711b654be3fcdd34fa391d2d274ce767"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:107ac60d1bfdc3edb53be75e2a52aff7481b92817cfdddd9b4519ccf0e54a6ff"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:00ba3c9818e33f1fa974693fb55d24cdc8ebafcb2e4207680669d8f8d7cca79a"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f0a47efb1dbef13af9c9a54a94a0b814902e547b7f21acb29434504d18f36e3a"}, + {file = "regex-2023.10.3-cp310-cp310-win32.whl", hash = "sha256:36362386b813fa6c9146da6149a001b7bd063dabc4d49522a1f7aa65b725c7ec"}, + {file = "regex-2023.10.3-cp310-cp310-win_amd64.whl", hash = "sha256:c65a3b5330b54103e7d21cac3f6bf3900d46f6d50138d73343d9e5b2900b2353"}, + {file = "regex-2023.10.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:90a79bce019c442604662d17bf69df99090e24cdc6ad95b18b6725c2988a490e"}, + {file = "regex-2023.10.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c7964c2183c3e6cce3f497e3a9f49d182e969f2dc3aeeadfa18945ff7bdd7051"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ef80829117a8061f974b2fda8ec799717242353bff55f8a29411794d635d964"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5addc9d0209a9afca5fc070f93b726bf7003bd63a427f65ef797a931782e7edc"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c148bec483cc4b421562b4bcedb8e28a3b84fcc8f0aa4418e10898f3c2c0eb9b"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d1f21af4c1539051049796a0f50aa342f9a27cde57318f2fc41ed50b0dbc4ac"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b9ac09853b2a3e0d0082104036579809679e7715671cfbf89d83c1cb2a30f58"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ebedc192abbc7fd13c5ee800e83a6df252bec691eb2c4bedc9f8b2e2903f5e2a"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d8a993c0a0ffd5f2d3bda23d0cd75e7086736f8f8268de8a82fbc4bd0ac6791e"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:be6b7b8d42d3090b6c80793524fa66c57ad7ee3fe9722b258aec6d0672543fd0"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4023e2efc35a30e66e938de5aef42b520c20e7eda7bb5fb12c35e5d09a4c43f6"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0d47840dc05e0ba04fe2e26f15126de7c755496d5a8aae4a08bda4dd8d646c54"}, + {file = "regex-2023.10.3-cp311-cp311-win32.whl", hash = "sha256:9145f092b5d1977ec8c0ab46e7b3381b2fd069957b9862a43bd383e5c01d18c2"}, + {file = "regex-2023.10.3-cp311-cp311-win_amd64.whl", hash = "sha256:b6104f9a46bd8743e4f738afef69b153c4b8b592d35ae46db07fc28ae3d5fb7c"}, + {file = "regex-2023.10.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bff507ae210371d4b1fe316d03433ac099f184d570a1a611e541923f78f05037"}, + {file = "regex-2023.10.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be5e22bbb67924dea15039c3282fa4cc6cdfbe0cbbd1c0515f9223186fc2ec5f"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a992f702c9be9c72fa46f01ca6e18d131906a7180950958f766c2aa294d4b41"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7434a61b158be563c1362d9071358f8ab91b8d928728cd2882af060481244c9e"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2169b2dcabf4e608416f7f9468737583ce5f0a6e8677c4efbf795ce81109d7c"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9e908ef5889cda4de038892b9accc36d33d72fb3e12c747e2799a0e806ec841"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12bd4bc2c632742c7ce20db48e0d99afdc05e03f0b4c1af90542e05b809a03d9"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bc72c231f5449d86d6c7d9cc7cd819b6eb30134bb770b8cfdc0765e48ef9c420"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bce8814b076f0ce5766dc87d5a056b0e9437b8e0cd351b9a6c4e1134a7dfbda9"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:ba7cd6dc4d585ea544c1412019921570ebd8a597fabf475acc4528210d7c4a6f"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b0c7d2f698e83f15228ba41c135501cfe7d5740181d5903e250e47f617eb4292"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5a8f91c64f390ecee09ff793319f30a0f32492e99f5dc1c72bc361f23ccd0a9a"}, + {file = "regex-2023.10.3-cp312-cp312-win32.whl", hash = "sha256:ad08a69728ff3c79866d729b095872afe1e0557251da4abb2c5faff15a91d19a"}, + {file = "regex-2023.10.3-cp312-cp312-win_amd64.whl", hash = "sha256:39cdf8d141d6d44e8d5a12a8569d5a227f645c87df4f92179bd06e2e2705e76b"}, + {file = "regex-2023.10.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4a3ee019a9befe84fa3e917a2dd378807e423d013377a884c1970a3c2792d293"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76066d7ff61ba6bf3cb5efe2428fc82aac91802844c022d849a1f0f53820502d"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe50b61bab1b1ec260fa7cd91106fa9fece57e6beba05630afe27c71259c59b"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fd88f373cb71e6b59b7fa597e47e518282455c2734fd4306a05ca219a1991b0"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3ab05a182c7937fb374f7e946f04fb23a0c0699c0450e9fb02ef567412d2fa3"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dac37cf08fcf2094159922edc7a2784cfcc5c70f8354469f79ed085f0328ebdf"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e54ddd0bb8fb626aa1f9ba7b36629564544954fff9669b15da3610c22b9a0991"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3367007ad1951fde612bf65b0dffc8fd681a4ab98ac86957d16491400d661302"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:16f8740eb6dbacc7113e3097b0a36065a02e37b47c936b551805d40340fb9971"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:f4f2ca6df64cbdd27f27b34f35adb640b5d2d77264228554e68deda54456eb11"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:39807cbcbe406efca2a233884e169d056c35aa7e9f343d4e78665246a332f597"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7eece6fbd3eae4a92d7c748ae825cbc1ee41a89bb1c3db05b5578ed3cfcfd7cb"}, + {file = "regex-2023.10.3-cp37-cp37m-win32.whl", hash = "sha256:ce615c92d90df8373d9e13acddd154152645c0dc060871abf6bd43809673d20a"}, + {file = "regex-2023.10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0f649fa32fe734c4abdfd4edbb8381c74abf5f34bc0b3271ce687b23729299ed"}, + {file = "regex-2023.10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9b98b7681a9437262947f41c7fac567c7e1f6eddd94b0483596d320092004533"}, + {file = "regex-2023.10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:91dc1d531f80c862441d7b66c4505cd6ea9d312f01fb2f4654f40c6fdf5cc37a"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82fcc1f1cc3ff1ab8a57ba619b149b907072e750815c5ba63e7aa2e1163384a4"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7979b834ec7a33aafae34a90aad9f914c41fd6eaa8474e66953f3f6f7cbd4368"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef71561f82a89af6cfcbee47f0fabfdb6e63788a9258e913955d89fdd96902ab"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd829712de97753367153ed84f2de752b86cd1f7a88b55a3a775eb52eafe8a94"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00e871d83a45eee2f8688d7e6849609c2ca2a04a6d48fba3dff4deef35d14f07"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:706e7b739fdd17cb89e1fbf712d9dc21311fc2333f6d435eac2d4ee81985098c"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cc3f1c053b73f20c7ad88b0d1d23be7e7b3901229ce89f5000a8399746a6e039"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6f85739e80d13644b981a88f529d79c5bdf646b460ba190bffcaf6d57b2a9863"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:741ba2f511cc9626b7561a440f87d658aabb3d6b744a86a3c025f866b4d19e7f"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e77c90ab5997e85901da85131fd36acd0ed2221368199b65f0d11bca44549711"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:979c24cbefaf2420c4e377ecd1f165ea08cc3d1fbb44bdc51bccbbf7c66a2cb4"}, + {file = "regex-2023.10.3-cp38-cp38-win32.whl", hash = "sha256:58837f9d221744d4c92d2cf7201c6acd19623b50c643b56992cbd2b745485d3d"}, + {file = "regex-2023.10.3-cp38-cp38-win_amd64.whl", hash = "sha256:c55853684fe08d4897c37dfc5faeff70607a5f1806c8be148f1695be4a63414b"}, + {file = "regex-2023.10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2c54e23836650bdf2c18222c87f6f840d4943944146ca479858404fedeb9f9af"}, + {file = "regex-2023.10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:69c0771ca5653c7d4b65203cbfc5e66db9375f1078689459fe196fe08b7b4930"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ac965a998e1388e6ff2e9781f499ad1eaa41e962a40d11c7823c9952c77123e"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c0e8fae5b27caa34177bdfa5a960c46ff2f78ee2d45c6db15ae3f64ecadde14"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6c56c3d47da04f921b73ff9415fbaa939f684d47293f071aa9cbb13c94afc17d"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ef1e014eed78ab650bef9a6a9cbe50b052c0aebe553fb2881e0453717573f52"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d29338556a59423d9ff7b6eb0cb89ead2b0875e08fe522f3e068b955c3e7b59b"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9c6d0ced3c06d0f183b73d3c5920727268d2201aa0fe6d55c60d68c792ff3588"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:994645a46c6a740ee8ce8df7911d4aee458d9b1bc5639bc968226763d07f00fa"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:66e2fe786ef28da2b28e222c89502b2af984858091675044d93cb50e6f46d7af"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:11175910f62b2b8c055f2b089e0fedd694fe2be3941b3e2633653bc51064c528"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:06e9abc0e4c9ab4779c74ad99c3fc10d3967d03114449acc2c2762ad4472b8ca"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fb02e4257376ae25c6dd95a5aec377f9b18c09be6ebdefa7ad209b9137b73d48"}, + {file = "regex-2023.10.3-cp39-cp39-win32.whl", hash = "sha256:3b2c3502603fab52d7619b882c25a6850b766ebd1b18de3df23b2f939360e1bd"}, + {file = "regex-2023.10.3-cp39-cp39-win_amd64.whl", hash = "sha256:adbccd17dcaff65704c856bd29951c58a1bd4b2b0f8ad6b826dbd543fe740988"}, + {file = "regex-2023.10.3.tar.gz", hash = "sha256:3fef4f844d2290ee0ba57addcec17eec9e3df73f10a2748485dfd6a3a188cc0f"}, ] [[package]] @@ -4741,4 +4807,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "0fd79094f1be19d817c7657a2e083fc1dd4ccac14f02732aff8d1ee9e7694afb" +content-hash = "f3ad3eeee416e7d6c329286a356112c7b5d041693833bc66b445c11bf273dcd7" diff --git a/pyproject.toml b/pyproject.toml index fe5dada..07ad751 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,8 @@ ftfy = "^6.1.1" tenacity = "^8.2.3" invoke = "^2.2.0" ray = "^2.7.0" +grpcio = "^1.59.0" +regex = "^2023.10.3" [tool.poetry.group.dev.dependencies] invoke = "^2.2.0" From bb3c04304256a5bdf7c0ae15cb58ace1d920d3c9 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Mon, 9 Oct 2023 10:25:28 -0700 Subject: [PATCH 2/4] Configure stopping criteria --- app/lesson/tasks.py | 17 ++++++++++++++++- app/llm/adaptors/oai.py | 16 ++++++++-------- app/llm/generators/concepts.py | 1 - app/llm/generators/lesson.py | 10 +++++++--- app/llm/generators/outline.py | 1 - app/llm/generators/topic.py | 1 - app/llm/llm.py | 22 ++++++++++++++++++---- app/llm/schemas.py | 2 +- app/settings.py | 4 +++- book_generator.py | 10 +++++++--- 10 files changed, 60 insertions(+), 24 deletions(-) diff --git a/app/lesson/tasks.py b/app/lesson/tasks.py index 7e30687..18e3491 100644 --- a/app/lesson/tasks.py +++ b/app/lesson/tasks.py @@ -22,6 +22,7 @@ async def generate_lesson( outline: List[str], revision: int, research_notes: List[ResearchNote] | None = None, + sections_per_generation: int = settings.SECTIONS_PER_GENERATION, ) -> List[AllLessonComponentData] | None: # Add numbers to the outline - needed for generating the lesson numbered_outline = outline @@ -62,9 +63,14 @@ async def generate_lesson( current_section = f"{last_section.strip()}\n\n{current_section_header.strip()}" current_section = f"{current_section}\n" + # When to stop generation + stop_section = None + if generated_sections + sections_per_generation < len(numbered_outline): + stop_section = numbered_outline[generated_sections + sections_per_generation] + # Filter research notes to save tokens, only keep notes relevant to the next 5 sections # Find the indices of the next sections - future_sections = set(list(range(generated_sections, len(numbered_outline)))[:5]) + future_sections = set(list(range(generated_sections, len(numbered_outline)))[:sections_per_generation]) selected_research_notes = None if research_notes is not None: selected_research_notes = [] @@ -84,6 +90,7 @@ async def generate_lesson( research_notes=selected_research_notes, include_examples=settings.INCLUDE_EXAMPLES, cache=use_cache, + stop_section=stop_section, ) new_components = [] new_component_keys = [] @@ -137,6 +144,7 @@ async def generate_single_lesson_chunk( research_notes: List[ResearchNote] | None, include_examples: bool, cache: bool, + stop_section: str | None = None, ) -> AsyncGenerator[List[AllLessonComponentData], None]: response = generate_lessons( numbered_outline, @@ -148,8 +156,15 @@ async def generate_single_lesson_chunk( research_notes=research_notes, include_examples=include_examples, cache=cache, + stop_section=stop_section, ) + section_start = f"---{ComponentNames.section}" + async for chunk in response: + # Remove the final section header from the chunk + # This happens when we hit the stop token + if chunk.strip().endswith(section_start): + chunk = chunk.strip()[:-len(section_start)] new_components = parse_lesson_markdown(chunk) yield new_components diff --git a/app/llm/adaptors/oai.py b/app/llm/adaptors/oai.py index 8ca56f4..de70c22 100644 --- a/app/llm/adaptors/oai.py +++ b/app/llm/adaptors/oai.py @@ -30,7 +30,7 @@ async def oai_chat_wrapped( history: List, temperature: float, max_tokens: int, - stop_tokens: Optional[List] = None, + stop_sequences: Optional[List] = None, model: str = settings.LLM_TYPE, ) -> AsyncGenerator[str, None]: response = await openai.ChatCompletion.acreate( @@ -39,7 +39,7 @@ async def oai_chat_wrapped( temperature=temperature, max_tokens=max_tokens, n=1, - stop=stop_tokens, + stop=stop_sequences, stream=True, ) async for chunk in response: @@ -54,7 +54,7 @@ async def oai_prompt_wrapped( prompt: str, temperature: float, max_tokens: int, - stop_tokens: Optional[List] = None, + stop_sequences: Optional[List] = None, model: str = settings.LLM_TYPE, ) -> AsyncGenerator[str, None]: response = await openai.Completion.acreate( @@ -63,7 +63,7 @@ async def oai_prompt_wrapped( temperature=temperature, max_tokens=max_tokens, n=1, - stop=stop_tokens, + stop=stop_sequences, stream=True, ) async for chunk in response: @@ -78,7 +78,7 @@ async def oai_prompt_response( temperature: float = settings.LLM_TEMPERATURE, timeout: int = settings.LLM_TIMEOUT, max_tokens: int = settings.LLM_MAX_RESPONSE_TOKENS, - stop_tokens=None, + stop_sequences=None, model: str = settings.LLM_TYPE, ) -> Optional[AsyncGenerator[LLMResponse, None]]: response_tokens = 0 @@ -88,7 +88,7 @@ async def oai_prompt_response( temperature, max_tokens, timeout=timeout, - stop_tokens=stop_tokens, + stop_sequences=stop_sequences, model=model, ) async for chunk in response: @@ -113,7 +113,7 @@ async def oai_chat_response( timeout: int = settings.LLM_TIMEOUT, max_tokens: int = settings.LLM_MAX_RESPONSE_TOKENS, history=None, - stop_tokens=None, + stop_sequences=None, model: str = settings.LLM_TYPE, ) -> Optional[AsyncGenerator[LLMResponse, None]]: current_message = {"role": "user", "content": prompt} @@ -130,7 +130,7 @@ async def oai_chat_response( temperature, max_tokens, timeout=timeout, - stop_tokens=stop_tokens, + stop_sequences=stop_sequences, model=model, ) async for chunk in response: diff --git a/app/llm/generators/concepts.py b/app/llm/generators/concepts.py index 19cd8b7..076ca30 100644 --- a/app/llm/generators/concepts.py +++ b/app/llm/generators/concepts.py @@ -26,7 +26,6 @@ class CourseGeneratedConcepts(BaseModel): temperature=0.7, max_tokens=256, timeout=40, - stop_tokens=None, prompt_type="concept", model=settings.LLM_INSTRUCT_TYPE, ) diff --git a/app/llm/generators/lesson.py b/app/llm/generators/lesson.py index 403e3ca..a57332d 100644 --- a/app/llm/generators/lesson.py +++ b/app/llm/generators/lesson.py @@ -12,8 +12,7 @@ lesson_settings = GenerationSettings( temperature=0.4, max_tokens=6000, - timeout=480, - stop_tokens=None, + timeout=1200, prompt_type="lesson", ) @@ -121,6 +120,7 @@ async def generate_lessons( include_examples: bool = True, update_after_chars: int = 500, cache: bool = True, + stop_section: str | None = None, ) -> AsyncGenerator[str, None]: prompt = lesson_prompt( outline, @@ -133,7 +133,11 @@ async def generate_lessons( ) text = "" - response = generate_response(prompt, lesson_settings, cache=cache, revision=revision) + stop_sequences = None + if stop_section is not None: + stop_sequences = [stop_section] + + response = generate_response(prompt, lesson_settings, cache=cache, revision=revision, stop_sequences=stop_sequences) chunk_len = 0 # Yield text in batches, to avoid creating too many DB models diff --git a/app/llm/generators/outline.py b/app/llm/generators/outline.py index 8914016..16be675 100644 --- a/app/llm/generators/outline.py +++ b/app/llm/generators/outline.py @@ -20,7 +20,6 @@ temperature=0.6, max_tokens=2048, timeout=60, - stop_tokens=None, prompt_type="outline", model=settings.LLM_INSTRUCT_TYPE, ) diff --git a/app/llm/generators/topic.py b/app/llm/generators/topic.py index 7803a14..30af892 100644 --- a/app/llm/generators/topic.py +++ b/app/llm/generators/topic.py @@ -14,7 +14,6 @@ temperature=0.9, max_tokens=512, timeout=40, - stop_tokens=None, prompt_type="topic", model=settings.LLM_INSTRUCT_TYPE, ) diff --git a/app/llm/llm.py b/app/llm/llm.py index 1e485e0..af7e271 100644 --- a/app/llm/llm.py +++ b/app/llm/llm.py @@ -26,16 +26,30 @@ async def generate_response( max_tries: int = 2, cache: bool = True, revision: int = 1, + stop_sequences: Optional[List[str]] = None, ) -> AsyncGenerator[str, None]: temperature = prompt_settings.temperature max_tokens = prompt_settings.max_tokens timeout = prompt_settings.timeout - stop_tokens = prompt_settings.stop_tokens + prompt_stops = prompt_settings.stop_sequences prompt_type = prompt_settings.prompt_type model = ( prompt_settings.model or settings.LLM_TYPE ) # Use default model if not specified + # Stop sequences for the llm + stops = [] + if prompt_stops is not None: + stops.extend(prompt_stops) + if stop_sequences is not None: + stops.extend(stop_sequences) + + # Only support up to 4 stop sequences + if len(stops) == 0: + stops = None + else: + stops = stops[:4] + # Remove utf-8 surrogate characters prompt = fix_unicode_text(prompt) @@ -84,7 +98,7 @@ async def generate_response( timeout, max_tokens, history, - stop_tokens, + stops, model=model, ) case "gpt-3.5-turbo-instruct": @@ -102,7 +116,7 @@ async def generate_response( temperature, timeout, max_tokens, - stop_tokens, + stops, model=model, ) case _: @@ -127,7 +141,7 @@ async def generate_response( temperature, timeout, max_tokens, - stop_tokens, + stops, model=model, ) break diff --git a/app/llm/schemas.py b/app/llm/schemas.py index 61f10e6..2c75094 100644 --- a/app/llm/schemas.py +++ b/app/llm/schemas.py @@ -12,7 +12,7 @@ class GenerationSettings(BaseModel): temperature: float max_tokens: int timeout: int - stop_tokens: Optional[List[str]] + stop_sequences: Optional[List[str]] prompt_type: str component_name: Optional[str] model: Optional[str] diff --git a/app/settings.py b/app/settings.py index 4d5cd7e..66a560c 100644 --- a/app/settings.py +++ b/app/settings.py @@ -19,6 +19,7 @@ class Settings(BaseSettings): # Content SECTIONS_PER_LESSON: int = 30 # Lower this to make books shorter + SECTIONS_PER_GENERATION: int = 5 # How many sections to generate in one prompt MAX_DOWNLOAD_SIZE: int = 6 * 1024 * 1024 # Max pdf size to download, 6 MB FINETUNED: bool = False # If we're using a finetuned textbook gen model INCLUDE_EXAMPLES: bool = ( @@ -36,7 +37,7 @@ class Settings(BaseSettings): } LLM_TEMPERATURE: float = 0.5 - LLM_TIMEOUT: int = 120 + LLM_TIMEOUT: int = 480 LLM_MAX_RESPONSE_TOKENS: int = 2048 OPENAI_KEY: str = "" OPENAI_BASE_URL: Optional[str] = None @@ -56,6 +57,7 @@ class Settings(BaseSettings): # General THREADS_PER_WORKER: int = 1 # How many threads to use per worker process to save RAM RAY_CACHE_PATH: Optional[str] = None # Where to save ray cache + RAY_DASHBOARD_HOST: str = "0.0.0.0" class Config: env_file = find_dotenv("local.env") diff --git a/book_generator.py b/book_generator.py index 9abdd6b..cf94587 100644 --- a/book_generator.py +++ b/book_generator.py @@ -51,7 +51,7 @@ def get_json_data_from_course(course: Course, extended_fields=False): return json.dumps(json_data) -async def generate_single_course(model, course_data: Dict | str, revision=1, outline_items=12): +async def generate_single_course(model, course_data: Dict | str, revision=1, outline_items=12, cache_only=False): components = ["exercise", "example"] outline = None @@ -69,6 +69,9 @@ async def generate_single_course(model, course_data: Dict | str, revision=1, out await asyncio.sleep(.001) # Sleep to avoid high CPU usage with many workers return course + if cache_only: + return None + if not outline: # Only generate outline if one was not passed in concepts = await create_course_concepts(course_name, revision) @@ -123,7 +126,7 @@ async def generate_single_course(model, course_data: Dict | str, revision=1, out async def _process_course(model, topic, args): try: - return await generate_single_course(model, topic, revision=args.revision) + return await generate_single_course(model, topic, revision=args.revision, cache_only=args.cache_only) except Exception as e: debug_print_trace() print(f"Unhandled error generating course: {e}") @@ -176,6 +179,7 @@ def to_iterator(obj_ids): parser.add_argument("--extended-fields", action="store_true", default=False, help="Include extended fields in output") parser.add_argument("--no_cache", action="store_true", default=False, help="Don't use the cache") parser.add_argument("--revision", type=int, default=1, help="Revision number for the course. Change this to avoid hitting cache if you want to regenerate a course.") + parser.add_argument("--cache-only", action="store_true", default=False, help="Only use the cache, don't generate any new courses") args = parser.parse_args() @@ -202,7 +206,7 @@ def to_iterator(obj_ids): total_processes = math.ceil(args.workers / settings.THREADS_PER_WORKER) func = process_courses - ray.init(num_cpus=total_processes, storage=settings.RAY_CACHE_PATH, _temp_dir=settings.RAY_CACHE_PATH) + ray.init(num_cpus=total_processes, storage=settings.RAY_CACHE_PATH, _temp_dir=settings.RAY_CACHE_PATH, dashboard_host=settings.RAY_DASHBOARD_HOST) model = SentenceTransformer("thenlper/gte-small") model_ref = ray.put(model) From a629bbf55d5d2f604c9fa2ddfe6ba66eeec95bc2 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Tue, 10 Oct 2023 12:58:25 -0700 Subject: [PATCH 3/4] Enable using pregenerated tocs --- README.md | 30 ++++++++++++++-- app/llm/adaptors/oai.py | 6 ++++ app/llm/examples/toc.json | 45 +++++++++++++++++++++++ app/llm/generators/concepts.py | 2 +- app/llm/generators/lesson.py | 25 ++++++++----- app/llm/generators/outline.py | 2 +- app/llm/generators/toc.py | 65 ++++++++++++++++++++++++++++++++++ app/llm/llm.py | 1 - app/llm/models.py | 1 + app/llm/templates/toc.jinja | 13 +++++++ app/services/generators/pdf.py | 2 +- book_generator.py | 33 ++++++++++------- toc_cleaner.py | 56 +++++++++++++++++++++++++++++ 13 files changed, 253 insertions(+), 28 deletions(-) create mode 100644 app/llm/examples/toc.json create mode 100644 app/llm/generators/toc.py create mode 100644 app/llm/templates/toc.jinja create mode 100644 toc_cleaner.py diff --git a/README.md b/README.md index 7c7b46d..f9dc88f 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ By default, this will use `gpt-3.5`. You can use `gpt-4` by setting the env var - Set the model name and max tokens in the `LLM_TYPES` setting. - Follow the instructions above for the retrieval setup. -The generator ideally needs a context length of up to `16k`, but you can get away with `12k` if you need to. +The generator ideally needs a context length of up to `16k`, but you can get away with `12k` if you need to. If you've finetuned your own model for textbook gen (based on the prompts cached in this repo), you can use the `FINETUNED` and `INCLUDE_EXAMPLES` settings to reduce token usage. ### Without retrieval @@ -73,6 +73,8 @@ Usage example: ## Generate textbooks +### From titles + This will take a file with a flat json list of topics, and generate one textbook per topic. The workers flag controls the number of parallel generations. Lower it if you hit rate limits. Usage example: @@ -83,7 +85,31 @@ You can also override settings with environment variables (instead of using `loc `LLM_TYPE=llama LLM_INSTRUCT_TYPE=llama LLM_EXTENDED_TYPE=llama OPENAI_KEY="llama" OPENAI_BASE_URL="https://vllm-api.com/v1" python book_generator.py topics.json books.jsonl --workers 10` -Note that courses are cached by default, so regenerating a course with the same name twice will not hit the API again. The cache is specific to each model and each topic. +You can see all options by running `python book_generator.py --help`. + +Note that courses are cached by default, so regenerating a course with the same name twice will not hit the API again. The cache is specific to each model and each topic. You can skip the cache by using the `--revision` option to specify a revision number for the courses. + +### From outlines + +You can also generate a book from an existing outline by creating a jsonl file with the following fields: + +- `topic` - The topic/title of the book +- `outline` - The outline of the book, as a flat json list. This needs to be in a specific format, see "clean table of contents" below. +- `queries` - Up to 2 search queries to use for retrieval. If you don't want to use retrieval, set this to an empty list. + +## Clean tables of contents + +This will take in a jsonl file with an existing table of contents and title, and process it into the correct format for book generation. + +Usage example: + +`python toc_cleaner.py toc.jsonl clean_toc.jsonl` + +`toc.jsonl` should have the following fields in each line: + +- `title` - The title of the book +- `toc` - a string containing the table of contents. This can be poorly formatted + # Extending diff --git a/app/llm/adaptors/oai.py b/app/llm/adaptors/oai.py index de70c22..9c26075 100644 --- a/app/llm/adaptors/oai.py +++ b/app/llm/adaptors/oai.py @@ -30,6 +30,7 @@ async def oai_chat_wrapped( history: List, temperature: float, max_tokens: int, + inner_timeout: int = settings.LLM_TIMEOUT, stop_sequences: Optional[List] = None, model: str = settings.LLM_TYPE, ) -> AsyncGenerator[str, None]: @@ -41,6 +42,7 @@ async def oai_chat_wrapped( n=1, stop=stop_sequences, stream=True, + request_timeout=inner_timeout, ) async for chunk in response: stream = chunk @@ -54,6 +56,7 @@ async def oai_prompt_wrapped( prompt: str, temperature: float, max_tokens: int, + inner_timeout: int = settings.LLM_TIMEOUT, stop_sequences: Optional[List] = None, model: str = settings.LLM_TYPE, ) -> AsyncGenerator[str, None]: @@ -65,6 +68,7 @@ async def oai_prompt_wrapped( n=1, stop=stop_sequences, stream=True, + request_timeout=inner_timeout, ) async for chunk in response: stream = chunk @@ -88,6 +92,7 @@ async def oai_prompt_response( temperature, max_tokens, timeout=timeout, + inner_timeout=timeout, stop_sequences=stop_sequences, model=model, ) @@ -130,6 +135,7 @@ async def oai_chat_response( temperature, max_tokens, timeout=timeout, + inner_timeout=timeout, stop_sequences=stop_sequences, model=model, ) diff --git a/app/llm/examples/toc.json b/app/llm/examples/toc.json new file mode 100644 index 0000000..01e2b67 --- /dev/null +++ b/app/llm/examples/toc.json @@ -0,0 +1,45 @@ +[ + { + "topic": "Python Programming for Beginners", + "draft_outline": "Introduction\n*What is Programming? *Why Python? *Historical Background of Python *Applications of Python Setting Up the Environment\n *Installing Python *Interactive Shell vs. Script Mode *Setting Up an IDE (e.g., PyCharm, VSCode)", + "json": { + "outline": [ + "1. Python Programming for Beginners", + "1.1. What is Programming?", + "1.2. Why Python?", + "1.3. Historical Background of Python", + "1.4. Applications of Python", + "2. Setting Up the Environment", + "2.1. Installing Python", + "2.2. Interactive Shell vs. Script Mode", + "2.3. Setting Up an IDE (e.g., PyCharm, VSCode)" + ], + "queries": [ + "Python programming beginner guide", + "Python programming introduction book" + ] + } + }, + { + "topic": "PLZ/SYS Programming Language Manual", + "draft_outline": "1. Introduction.- 1.1 PLZ/SYS objectives.- 2. Summary Of The Language.- 2.1 Data and Statements.- 2.2 The Construction of a Program.- 3. Notation, Terminology, And Vocabulary.- 3.1 Vocabulary.- 3.2 Lexical Structure.- 4. Identifiers And Literal Constants.- PLZ/SYS Grammar. Conclusion. References.", + "json": { + "outline": [ + "1. Introduction", + "1.1 PLZ/SYS objectives", + "2. Summary Of The Language", + "2.1 Data and Statements", + "2.2 The Construction of a Program", + "3. Notation, Terminology, And Vocabulary", + "3.1 Vocabulary", + "3.2 Lexical Structure", + "4. Identifiers And Literal Constants", + "5. PLZ/SYS Grammar" + ], + "queries": [ + "PLZ/SYS programming language overview", + "Best practices for Structured Statements in programming" + ] + } + } +] \ No newline at end of file diff --git a/app/llm/generators/concepts.py b/app/llm/generators/concepts.py index 076ca30..b034059 100644 --- a/app/llm/generators/concepts.py +++ b/app/llm/generators/concepts.py @@ -25,7 +25,7 @@ class CourseGeneratedConcepts(BaseModel): concept_settings = GenerationSettings( temperature=0.7, max_tokens=256, - timeout=40, + timeout=1200, prompt_type="concept", model=settings.LLM_INSTRUCT_TYPE, ) diff --git a/app/llm/generators/lesson.py b/app/llm/generators/lesson.py index a57332d..3cc8ab9 100644 --- a/app/llm/generators/lesson.py +++ b/app/llm/generators/lesson.py @@ -8,6 +8,7 @@ from app.llm.llm import GenerationSettings, generate_response from app.llm.prompts import build_prompt from app.settings import settings +from copy import deepcopy lesson_settings = GenerationSettings( temperature=0.4, @@ -36,8 +37,6 @@ def lesson_prompt( with open(os.path.join(settings.EXAMPLE_JSON_DIR, "lesson.json")) as f: examples = json.load(f) - rendered_outline = "\n".join(outline).strip() - # Set default components if none are provided if not components: components = list(get_args(settings.VALID_GENERATED_COMPONENTS)) @@ -65,6 +64,21 @@ def lesson_prompt( component_extras = [COMPONENT_EXTRAS.get(c, None) for c in components] component_extras = [c for c in component_extras if c is not None] + sections_to_author = len(outline) - current_section_index + outline_items_to_author = outline[ + current_section_index: current_section_index + sections_to_author + ] + outline_items_to_author_str = ",".join(outline_items_to_author) + outline_stop_item = outline_items_to_author[-1] + + selected_outline = deepcopy(outline) + if len(outline) > settings.SECTIONS_PER_LESSON: + surround = min(settings.SECTIONS_PER_LESSON // 2, 10) + start_item = max(current_section_index - surround, 0) + end_item = min(current_section_index + sections_to_author + surround, len(outline)) + selected_outline = selected_outline[start_item:end_item] + + rendered_outline = "\n".join(selected_outline).strip() items = [ ("Table of contents\n", rendered_outline), ] @@ -82,13 +96,6 @@ def lesson_prompt( input = OrderedDict(items) - sections_to_author = len(outline) - current_section_index - outline_items_to_author = outline[ - current_section_index: current_section_index + sections_to_author - ] - outline_items_to_author_str = ",".join(outline_items_to_author) - outline_stop_item = outline_items_to_author[-1] - section_str = "section" if sections_to_author == 1 else "sections" prompt = build_prompt( diff --git a/app/llm/generators/outline.py b/app/llm/generators/outline.py index 16be675..4dfa17c 100644 --- a/app/llm/generators/outline.py +++ b/app/llm/generators/outline.py @@ -19,7 +19,7 @@ outline_settings = GenerationSettings( temperature=0.6, max_tokens=2048, - timeout=60, + timeout=1200, prompt_type="outline", model=settings.LLM_INSTRUCT_TYPE, ) diff --git a/app/llm/generators/toc.py b/app/llm/generators/toc.py new file mode 100644 index 0000000..a47d9b8 --- /dev/null +++ b/app/llm/generators/toc.py @@ -0,0 +1,65 @@ +import json +import os +from collections import OrderedDict +from copy import deepcopy +from json import JSONDecodeError +from typing import List + +import ftfy +from pydantic import BaseModel + +from app.llm.exceptions import GenerationError +from app.llm.llm import GenerationSettings, generate_response +from app.llm.prompts import build_prompt +from app.settings import settings +from app.util import extract_only_json_dict +from app.llm.adaptors.oai import oai_tokenize_prompt + + +class GeneratedTOC(BaseModel): + topic: str + outline: List[str] + queries: List[str] + + +toc_settings = GenerationSettings( + temperature=0.5, + max_tokens=1024, + timeout=1200, + prompt_type="toc", + model=settings.LLM_TYPE, +) + + +def toc_prompt(topic: str, toc: str, include_examples=True) -> str: + with open(os.path.join(settings.EXAMPLE_JSON_DIR, "toc.json")) as f: + examples = json.load(f) + input = OrderedDict([ + ("topic", topic), + ("toc", toc), + ]) + prompt = build_prompt("toc", input, examples, include_examples=include_examples) + return prompt + + +async def generate_tocs(topic: str, draft_toc: str, include_examples: bool = True) -> GeneratedTOC: + prompt = toc_prompt(topic, draft_toc, include_examples=include_examples) + text = "" + + settings_inst = deepcopy(toc_settings) + settings_inst.max_tokens = oai_tokenize_prompt(draft_toc) + 512 # Max tokens to generate + + response = generate_response(prompt, settings_inst) + async for chunk in response: + text += chunk + try: + text = extract_only_json_dict(text) + text = str(ftfy.fix_text(text)) + data = json.loads(text.strip()) + toc = data["outline"] + queries = data["queries"] + except (JSONDecodeError, IndexError) as e: + raise GenerationError(e) + + model = GeneratedTOC(topic=topic, outline=toc, queries=queries) + return model diff --git a/app/llm/llm.py b/app/llm/llm.py index af7e271..225498e 100644 --- a/app/llm/llm.py +++ b/app/llm/llm.py @@ -156,7 +156,6 @@ async def generate_response( full_text = "" async for chunk in response: text = chunk.text - response_tokens = chunk.tokens yield text full_text += text diff --git a/app/llm/models.py b/app/llm/models.py index 6b46b9f..af1c4a6 100644 --- a/app/llm/models.py +++ b/app/llm/models.py @@ -10,6 +10,7 @@ class PromptTypes(str, BaseEnum): outline = "outline" topic = "topic" title = "title" + toc = "toc" class Prompt(BaseDBModel, table=True): diff --git a/app/llm/templates/toc.jinja b/app/llm/templates/toc.jinja new file mode 100644 index 0000000..81c5679 --- /dev/null +++ b/app/llm/templates/toc.jinja @@ -0,0 +1,13 @@ +{% extends "template.jinja" %} + +{%block content %} +You're cleaning up the draft outline for a textbook. You need to turn the draft outline into a final outline. The final format should be a flat json list, with each chapter numbered like 1, 2, or 3, and each section numbered like 1.1, or 1.2. If you need to, you can denote subsections like 1.1.1, or 1.1.2. + +Remove any parts of the draft that are not related to the main content of the book. For example, remove the preface, references, and the glossary. + +The final outline will be used to write a high quality textbook for college students. If the draft outline is insufficient to produce a high quality college textbook, feel free to add to it to make it sufficient. The final outline should have at least 20 total entries in it. + +Also return up to two Google Search queries that you can run to get additional resources on the topic while writing the textbook. The queries should pertain to the topic of the textbook and the outline. The queries will help you get information about topics you're not as familiar with, but the topics should not be so specific that no results can be found for them. + +Return the outline and queries in JSON format. Do not include more than 2 items in the queries. Only respond with valid JSON. +{% endblock %} diff --git a/app/services/generators/pdf.py b/app/services/generators/pdf.py index f780a2a..85e7edd 100644 --- a/app/services/generators/pdf.py +++ b/app/services/generators/pdf.py @@ -42,7 +42,7 @@ class PDFSearchResult(BaseModel): query: str -async def search_pdfs(queries: List[str], max_count=3) -> List[PDFSearchResult]: +async def search_pdfs(queries: List[str], max_count=10) -> List[PDFSearchResult]: coroutines = [search_pdf(query, max_count) for query in queries] results = await asyncio.gather(*coroutines) results = list(chain.from_iterable(results)) diff --git a/book_generator.py b/book_generator.py index cf94587..3cb48bc 100644 --- a/book_generator.py +++ b/book_generator.py @@ -91,16 +91,16 @@ async def generate_single_course(model, course_data: Dict | str, revision=1, out outline = outline[1:] outline = renumber_outline(outline) - context = None - if queries is not None: - try: - # Up to one retrieved passage per outline item - # Remove numbers from outline for use in retrieval - context_outline = [item.split(" ", 1)[-1] for item in outline] - context = await query_course_context(model, queries, context_outline) - except Exception as e: - debug_print_trace() - print(f"Error generating context for {course_name}: {e}") + context = None + if queries is not None: + try: + # Up to one retrieved passage per outline item + # Remove numbers from outline for use in retrieval + context_outline = [item.split(" ", 1)[-1] for item in outline] + context = await query_course_context(model, queries, context_outline) + except Exception as e: + debug_print_trace() + print(f"Error generating context for {course_name}: {e}") components = await generate_lesson(course_name, components, outline, revision, research_notes=context) if components is None: @@ -156,7 +156,15 @@ def process_course(model, course, args): def load_topics(in_file: str): with open(os.path.join(settings.DATA_DIR, in_file)) as f: - topics = json.load(f) + if in_file.endswith(".json"): + topics = json.load(f) + elif in_file.endswith(".jsonl"): + lines = list(f) + topics = [] + for line in lines: + topics.append(json.loads(line)) + else: + raise Exception(f"Unknown file type for {in_file}") random.seed(1) random.shuffle(topics) @@ -172,12 +180,11 @@ def to_iterator(obj_ids): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Given a topic file, generate synthetic books.") - parser.add_argument("in_file", help="Input filename (flat json list of topics, or dictionary with keys topic, queries, and outline). One file or a comma-separated list of files.") + parser.add_argument("in_file", help="Input filename (flat json list of topics, or jsonl file with dictionaries with keys topic, queries, and outline). One file or a comma-separated list of files.") parser.add_argument("out_file", help="Output filename (jsonl)") parser.add_argument("--max", type=int, default=None, help="Maximum number of courses to generate") parser.add_argument("--workers", type=int, default=5, help="Number of workers to use") parser.add_argument("--extended-fields", action="store_true", default=False, help="Include extended fields in output") - parser.add_argument("--no_cache", action="store_true", default=False, help="Don't use the cache") parser.add_argument("--revision", type=int, default=1, help="Revision number for the course. Change this to avoid hitting cache if you want to regenerate a course.") parser.add_argument("--cache-only", action="store_true", default=False, help="Only use the cache, don't generate any new courses") diff --git a/toc_cleaner.py b/toc_cleaner.py new file mode 100644 index 0000000..6bffadd --- /dev/null +++ b/toc_cleaner.py @@ -0,0 +1,56 @@ +import argparse + +from tqdm.contrib.concurrent import process_map + +from app.llm.exceptions import GenerationError, InvalidRequestError +from app.settings import settings +import json +import os +from app.llm.generators.toc import generate_tocs +import asyncio + +from app.util import debug_print_trace + + +async def generate_tocs_async(topic, toc): + final = await generate_tocs(topic, toc) + return final + + +def generate_tocs_sync(topic, toc): + try: + return asyncio.run(generate_tocs_async(topic, toc)) + except (GenerationError, InvalidRequestError): + debug_print_trace() + print(f"Error generating toc for {topic}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Clean up real ToCs from books.") + parser.add_argument("in_file", help="Input tocs and titles in jsonl format. Should be a list of json objects with the keys toc (str) and title (str).", type=str) + parser.add_argument("out_file", help="Output filename in jsonl format (list of json objects)") + parser.add_argument("--max", type=int, default=None, help="Maximum number of tocs to generate") + parser.add_argument("--workers", type=int, default=5, help="Number of workers to use") + args = parser.parse_args() + + topics = [] + with open(os.path.join(settings.DATA_DIR, args.in_file)) as f: + lines = list(f) + + for line in lines: + topics.append(json.loads(line)) + + if args.max is not None: + topics = topics[:args.max] + + new_tocs = process_map(generate_tocs_sync, [t["title"] for t in topics], [t["toc"] for t in topics], max_workers=args.workers, chunksize=1) + new_tocs = [t for t in new_tocs if t is not None] + + json_tocs = [t.dict() for t in new_tocs] + with open(os.path.join(settings.DATA_DIR, args.out_file), "w+") as f: + for line in json_tocs: + f.write(json.dumps(line)) + f.write("\n") + + print(f"Generated {len(json_tocs)} tocs") + From 7ceae0d6700f9d4a6d21d892e6cc1b9758ed47a4 Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Wed, 11 Oct 2023 15:44:56 -0700 Subject: [PATCH 4/4] Improve scalability --- app/course/models.py | 6 +++--- app/lesson/output.py | 14 +++++++----- app/llm/generators/toc.py | 7 ++++-- app/services/dependencies.py | 18 +++++++++++----- app/services/generators/pdf.py | 19 +++++++++-------- app/settings.py | 1 + book_generator.py | 39 ++++++++++++++++++++-------------- 7 files changed, 64 insertions(+), 40 deletions(-) diff --git a/app/course/models.py b/app/course/models.py index 17a464a..d8ddcfc 100644 --- a/app/course/models.py +++ b/app/course/models.py @@ -41,9 +41,9 @@ async def load_cached_course(model: str, topic: str, revision: int): select(Course).where(Course.topic == topic, Course.model == model, Course.version == revision) ) course = query.all() - if len(course) == 0: - return None - course = course[0] + if len(course) == 0: + return None + course = course[0] if course.context is not None: course.context = [ResearchNote(**json.loads(v)) for v in course.context] diff --git a/app/lesson/output.py b/app/lesson/output.py index a48c68d..1a3ac17 100644 --- a/app/lesson/output.py +++ b/app/lesson/output.py @@ -24,12 +24,14 @@ def render_components_to_output_markdown( ) ) case ComponentNames.section: - if not component.markdown.startswith("#"): - component.markdown = f"# {component.markdown}" - tuples.append((component.type, component.markdown)) + markdown_data = component.markdown.strip() + if not markdown_data.startswith("#"): + markdown_data = f"# {markdown_data}" + tuples.append((component.type, markdown_data)) case ComponentNames.text: + markdown_data = component.markdown.strip() tuples.append( - (component.type, remove_section_paragraphs(component.markdown)) + (component.type, remove_section_paragraphs(markdown_data)) ) case _: tuples.append((component.type, component.markdown)) @@ -50,4 +52,6 @@ def remove_section_paragraphs(text): paragraphs.pop() # Reconstruct the text from the remaining paragraphs - return "\n".join(paragraphs).strip() + replaced = "\n".join(paragraphs).strip() + replaced = re.sub(r"\n\n+", "\n\n", replaced) + return replaced diff --git a/app/llm/generators/toc.py b/app/llm/generators/toc.py index a47d9b8..d55c8ca 100644 --- a/app/llm/generators/toc.py +++ b/app/llm/generators/toc.py @@ -42,12 +42,15 @@ def toc_prompt(topic: str, toc: str, include_examples=True) -> str: return prompt -async def generate_tocs(topic: str, draft_toc: str, include_examples: bool = True) -> GeneratedTOC: +async def generate_tocs(topic: str, draft_toc: str, include_examples: bool = True) -> GeneratedTOC | None: prompt = toc_prompt(topic, draft_toc, include_examples=include_examples) text = "" settings_inst = deepcopy(toc_settings) - settings_inst.max_tokens = oai_tokenize_prompt(draft_toc) + 512 # Max tokens to generate + try: + settings_inst.max_tokens = oai_tokenize_prompt(draft_toc) + 512 # Max tokens to generate + except Exception: + return response = generate_response(prompt, settings_inst) async for chunk in response: diff --git a/app/services/dependencies.py b/app/services/dependencies.py index c919edf..e0ce7f0 100644 --- a/app/services/dependencies.py +++ b/app/services/dependencies.py @@ -1,3 +1,5 @@ +from typing import List, Optional + from sqlmodel import select from sqlmodel.ext.asyncio.session import AsyncSession @@ -5,11 +7,17 @@ from app.services.models import ScrapedData, ServiceResponse -async def get_stored_url(db: AsyncSession, url: str) -> str | None: - query = await db.exec(select(ScrapedData).where(ScrapedData.source == url)) - stored_url = query.first() - if stored_url: - return stored_url.uploaded +async def get_stored_urls(urls: List[str]) -> List[Optional[str]]: + async with get_session() as db: + query = await db.exec(select(ScrapedData).where(ScrapedData.source.in_(urls))) + stored_urls = query.all() + return_data = {} + for url in urls: + return_data[url] = None + for stored_url in stored_urls: + if url == stored_url.source: + return_data[url] = stored_url.uploaded + return [return_data[url] for url in urls] async def get_service_response_model(name: str, hex: str): diff --git a/app/services/generators/pdf.py b/app/services/generators/pdf.py index 85e7edd..7bbce2c 100644 --- a/app/services/generators/pdf.py +++ b/app/services/generators/pdf.py @@ -12,7 +12,7 @@ from app.db.session import get_session from app.services.adaptors.serpapi import serpapi_pdf_search_settings from app.services.adaptors.serply import serply_pdf_search_settings -from app.services.dependencies import get_stored_url +from app.services.dependencies import get_stored_urls from app.services.exceptions import ProcessingError from app.services.models import store_scraped_data from app.services.network import download_and_save @@ -42,9 +42,14 @@ class PDFSearchResult(BaseModel): query: str -async def search_pdfs(queries: List[str], max_count=10) -> List[PDFSearchResult]: +async def search_pdfs(queries: List[str], max_count=3) -> List[PDFSearchResult]: coroutines = [search_pdf(query, max_count) for query in queries] - results = await asyncio.gather(*coroutines) + + # Run queries sequentially + results = [] + for routine in coroutines: + results.append(await routine) + results = list(chain.from_iterable(results)) # Filter results to only unique links @@ -99,9 +104,6 @@ async def search_pdf(query: str, max_count) -> List[PDFSearchResult]: async def download_and_parse_pdfs( search_results: List[PDFSearchResult], ) -> List[PDFData]: - # Query if PDFs have already been downloaded and stored - pdf_paths = [] - # Deduplicate links deduped_search_results = [] seen_links = set() @@ -110,9 +112,8 @@ async def download_and_parse_pdfs( deduped_search_results.append(search_result) seen_links.add(search_result.link) - async with get_session() as db: - for search_result in deduped_search_results: - pdf_paths.append(await get_stored_url(db, search_result.link)) + links = [search_result.link for search_result in deduped_search_results] + pdf_paths = await get_stored_urls(links) coroutines = [ download_and_parse_pdf(search_result, pdf_path) for search_result, pdf_path in zip(deduped_search_results, pdf_paths) diff --git a/app/settings.py b/app/settings.py index 66a560c..7d3afec 100644 --- a/app/settings.py +++ b/app/settings.py @@ -58,6 +58,7 @@ class Settings(BaseSettings): THREADS_PER_WORKER: int = 1 # How many threads to use per worker process to save RAM RAY_CACHE_PATH: Optional[str] = None # Where to save ray cache RAY_DASHBOARD_HOST: str = "0.0.0.0" + RAY_CORES_PER_WORKER = .5 # How many cpu cores to allocate per worker class Config: env_file = find_dotenv("local.env") diff --git a/book_generator.py b/book_generator.py index 3cb48bc..9f8b9c8 100644 --- a/book_generator.py +++ b/book_generator.py @@ -1,7 +1,7 @@ -import asyncio import math from typing import Optional, Dict import argparse +import asyncio from sentence_transformers import SentenceTransformer from tqdm import tqdm @@ -66,7 +66,7 @@ async def generate_single_course(model, course_data: Dict | str, revision=1, out course = await load_cached_course(settings.LLM_TYPE, course_name, revision) if course is not None: - await asyncio.sleep(.001) # Sleep to avoid high CPU usage with many workers + await asyncio.sleep(0.01) # small sleep to avoid excess db load return course if cache_only: @@ -137,7 +137,7 @@ async def _process_courses(model, courses, args): return await asyncio.gather(*processes) -@ray.remote +@ray.remote(num_cpus=settings.RAY_CORES_PER_WORKER) def process_courses(model, courses, args): try: return asyncio.run(_process_courses(model, courses, args)) @@ -145,7 +145,8 @@ def process_courses(model, courses, args): debug_print_trace() print(f"Unhandled error generating courses: {e}") -@ray.remote + +@ray.remote(num_cpus=settings.RAY_CORES_PER_WORKER) def process_course(model, course, args): try: return asyncio.run(_process_course(model, course, args)) @@ -172,12 +173,6 @@ def load_topics(in_file: str): return topics -def to_iterator(obj_ids): - while obj_ids: - done, obj_ids = ray.wait(obj_ids) - yield ray.get(done[0]) - - if __name__ == "__main__": parser = argparse.ArgumentParser(description="Given a topic file, generate synthetic books.") parser.add_argument("in_file", help="Input filename (flat json list of topics, or jsonl file with dictionaries with keys topic, queries, and outline). One file or a comma-separated list of files.") @@ -203,27 +198,39 @@ def to_iterator(obj_ids): # Everything is cached, so exact duplicates will result in the same output topics = exact_deduplicate(topics) - total_processes = args.workers + total_processes = math.ceil(args.workers * settings.RAY_CORES_PER_WORKER) func = process_course if settings.THREADS_PER_WORKER > 1: # group topics into batches of settings.THREADS_PER_WORKER topics = [topics[i:i + settings.THREADS_PER_WORKER] for i in range(0, len(topics), settings.THREADS_PER_WORKER)] - total_processes = math.ceil(args.workers / settings.THREADS_PER_WORKER) + total_processes = math.ceil(total_processes / settings.THREADS_PER_WORKER) func = process_courses - ray.init(num_cpus=total_processes, storage=settings.RAY_CACHE_PATH, _temp_dir=settings.RAY_CACHE_PATH, dashboard_host=settings.RAY_DASHBOARD_HOST) + ray.init( + num_cpus=total_processes, + storage=settings.RAY_CACHE_PATH, + _temp_dir=settings.RAY_CACHE_PATH, + dashboard_host=settings.RAY_DASHBOARD_HOST + ) - model = SentenceTransformer("thenlper/gte-small") + model = SentenceTransformer("TaylorAI/gte-tiny") model_ref = ray.put(model) print(f"Generating {len(topics)} course batches with {total_processes} processes from filename(s) {in_files}") futures = [func.remote(model_ref, batch, args) for batch in topics] + # Run all ray tasks courses = [] - for x in tqdm(to_iterator(futures), total=len(futures)): - courses.append(x) + progress_bar = tqdm(total=len(futures)) + while len(futures) > 0: + finished, futures = ray.wait( + futures, timeout=7.0 + ) + course = ray.get(finished) + courses.extend(course) + progress_bar.update(len(course)) if settings.THREADS_PER_WORKER > 1: # Flatten courses list