diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py index 314d6c712..74fa3ee73 100644 --- a/dspy/signatures/signature.py +++ b/dspy/signatures/signature.py @@ -35,7 +35,9 @@ def __call__(cls, *args, **kwargs): # noqa: ANN002 def __new__(mcs, signature_name, bases, namespace, **kwargs): # noqa: N804 # Set `str` as the default type for all fields raw_annotations = namespace.get("__annotations__", {}) - for name, _field in namespace.items(): + for name, field in namespace.items(): + if not isinstance(field, FieldInfo): + continue # Don't add types to non-field attributes if not name.startswith("__") and name not in raw_annotations: raw_annotations[name] = str namespace["__annotations__"] = raw_annotations @@ -272,9 +274,8 @@ def make_signature( def _parse_signature(signature: str) -> Tuple[Type, Field]: - pattern = r"^\s*[\w\s,:]+\s*->\s*[\w\s,:]+\s*$" - if not re.match(pattern, signature): - raise ValueError(f"Invalid signature format: '{signature}'") + if signature.count("->") != 1: + raise ValueError(f"Invalid signature format: '{signature}', must contain exactly one '->'.") fields = {} inputs_str, outputs_str = map(str.strip, signature.split("->")) diff --git a/dspy/teleprompt/signature_opt_typed.py b/dspy/teleprompt/signature_opt_typed.py index 1921f8cd1..c86d297c1 100644 --- a/dspy/teleprompt/signature_opt_typed.py +++ b/dspy/teleprompt/signature_opt_typed.py @@ -1,4 +1,5 @@ import textwrap +from dataclasses import dataclass from typing import Generic, Literal, TypeVar import pydantic @@ -99,28 +100,48 @@ class GenerateInstructionInitial(Signature, Generic[T]): return GenerateInstructionInitial -class GenerateSignature(dspy.Signature, Generic[T]): - __doc__ = textwrap.dedent("""\ - You are an instruction optimizer for large language models. +def generate_with_avoidance(signatures_to_avoid: list[BaseModel]) -> type[Signature]: + class GenerateSignature(dspy.Signature, Generic[T]): + __doc__ = textwrap.dedent("""\ + You are an instruction optimizer for large language models. + + I will give some task instructions I've tried, along with their corresponding validation scores. + - The instructions are arranged in order based on their scores, where higher scores indicate better quality. + - Your task is to propose a new instruction that will lead a good language model to perform the task even better. + - Be creative, and think out of the box. + - Don't repeat instructions, descriptions and prefixes that have already been attempted. + """) + + analysis: str = OutputField(desc="Consider what made the previous instructions good or bad.") + proposed_signature: T = OutputField(desc="A signature that will likely lead to a high score.") + score: float = OutputField( + desc="The expected score for the new signature. Don't write anything after this number.", + ) + + @pydantic.field_validator("proposed_signature") + @classmethod + def check_signature_not_attempted(cls, s: T) -> T: + if s in signatures_to_avoid: + raise ValueError("Never propose a signature already in the list above.") + return s - I will give some task instructions I've tried, along with their corresponding validation scores. - - The instructions are arranged in order based on their scores, where higher scores indicate better quality. - - Your task is to propose a new instruction that will lead a good language model to perform the task even better. - - Be creative, and think out of the box. - - Don't repeat instructions, descriptions and prefixes that have already been attempted. - """) + return GenerateSignature - analysis: str = OutputField(desc="Consider what made the previous instructions good or bad.") - proposed_signature: T = OutputField(desc="A signature that will likely lead to a high score.") - score: float = OutputField(desc="The expected score for the new signature. Don't write anything after this number.") + +@dataclass +class OptimizerResult: + program: dspy.Program + signatures: list[dict[str, Signature]] + scores: list[float] def optimize_signature( student, evaluator, n_iterations=10, - strategy: Literal["best", "last"] = "best", - sorted_order: Literal["increasing", "decreasing"] = "increasing", + sorted_order: Literal["increasing", "decreasing", "chronological"] = "increasing", + strategy: Literal["last", "best"] = "best", + max_examples=20, # Formerly part of the constructor prompt_model=None, initial_prompts=2, @@ -139,10 +160,12 @@ def optimize_signature( The evaluator to use to score the program. n_iterations : int, optional The number of iterations to run, by default 10 - strategy : Literal["best", "last"], optional - The strategy to use to select the final program, by default "best" - sorted_order : Literal["increasing", "decreasing"], optional + max_examples : int, optional + The maximum number of examples to use for the evaluator, by default 20 + sorted_order : Literal["increasing", "decreasing", "chronological"], optional The order in which to sort the scores, by default "increasing" + strategy : Literal["last", "best"], optional + The strategy to use to select the final program, by default "best" prompt_model : dspy.LanguageModel, optional The language model to use to generate prompts, by default None initial_prompts : int, optional @@ -222,16 +245,22 @@ def optimize_signature( # TODO: Parallelize this for name, _p in named_predictors: SignatureInfo = type(candidates[name][0]) # noqa: N806 - generator = TypedPredictor(GenerateSignature[SignatureInfo]) - - demos = [ - dspy.Example( - proposed_signature=info, - score=sc, - ) - for info, sc in zip(candidates[name], scores) - ] - demos.sort(key=(lambda x: x.score), reverse=(sorted_order == "decreasing")) + + demos = [dspy.Example(proposed_signature=info, score=sc) for info, sc in zip(candidates[name], scores)] + if sorted_order == "chronological": + demos = demos[-max_examples:] + elif sorted_order == "increasing": + demos.sort(key=(lambda x: x.score), reverse=False) + demos = demos[-max_examples:] + elif sorted_order == "decreasing": + demos.sort(key=(lambda x: x.score), reverse=True) + demos = demos[:max_examples] + else: + raise ValueError(f"Invalid sorted_order: {sorted_order}") + + # We can only tell the LM to avoid the signatures we are actually giving it as demos. + avoid = [ex.proposed_signature for ex in demos] + generator = TypedPredictor(generate_with_avoidance(avoid)[SignatureInfo]) generator.predictor.demos = demos if verbose: @@ -240,12 +269,16 @@ def optimize_signature( candidates[name].append(new_signature) if strategy == "last": - return module - - if strategy == "best": + pass + elif strategy == "best": i = scores.index(max(scores)) for name, p in named_predictors: p.signature = candidates[name][i].to_signature() - return module + else: + raise ValueError(f"Invalid strategy: {strategy}") - raise ValueError(f"Invalid strategy: {strategy}") + return OptimizerResult( + program=module, + signatures=[{name: sigs[i].to_signature()} for name, sigs in candidates.items() for i in range(n_iterations)], + scores=scores, + ) diff --git a/examples/functional/signature_opt_typed.ipynb b/examples/functional/signature_opt_typed.ipynb index 7447a965e..feab8d635 100644 --- a/examples/functional/signature_opt_typed.ipynb +++ b/examples/functional/signature_opt_typed.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -17,9 +17,18 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/homebrew/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "import dspy\n", "turbo = dspy.OpenAI(model='gpt-3.5-turbo', max_tokens=4000)\n", @@ -29,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -40,7 +49,7 @@ ")" ] }, - "execution_count": 4, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -51,7 +60,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -60,7 +69,7 @@ "(20, 50)" ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -80,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -93,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -101,7 +110,7 @@ "output_type": "stream", "text": [ "Found 1 typed predictors to optimize.\n", - "Generating 4 initial signatures for base...\n", + "Generating 6 initial signatures for base...\n", "\n", "================================================================================\n", "Running eval iteration 0...\n" @@ -111,8 +120,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 4290.32it/s]\n", - "/Users/ahle/repos/dspy/dspy/evaluate/evaluate.py:142: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 3086.59it/s]\n", + "/Users/ahle/repos/dspy/dspy/evaluate/evaluate.py:145: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", " df = df.applymap(truncate_cell)\n" ] }, @@ -130,14 +139,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:02<00:00, 22.35it/s]\n" + "Average Metric: 1 / 50 (2.0): 100%|██████████| 50/50 [00:00<00:00, 1268.65it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 16 / 50 (32.0%)\n", + "Average Metric: 1 / 50 (2.0%)\n", "\n", "================================================================================\n", "Running eval iteration 2...\n" @@ -147,14 +156,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 19 / 50 (38.0): 100%|██████████| 50/50 [00:04<00:00, 10.28it/s]\n" + "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:00<00:00, 1031.35it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 19 / 50 (38.0%)\n", + "Average Metric: 17 / 50 (34.0%)\n", "\n", "================================================================================\n", "Running eval iteration 3...\n" @@ -164,14 +173,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 11 / 50 (22.0): 100%|██████████| 50/50 [00:05<00:00, 8.63it/s]\n" + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 1364.88it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 11 / 50 (22.0%)\n", + "Average Metric: 16 / 50 (32.0%)\n", "\n", "================================================================================\n", "Running eval iteration 4...\n" @@ -181,15 +190,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:02<00:00, 24.53it/s]\n" + "Average Metric: 6 / 50 (12.0): 100%|██████████| 50/50 [00:00<00:00, 892.68it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 15 / 50 (30.0%)\n", - "Generating new signature for base...\n", + "Average Metric: 6 / 50 (12.0%)\n", "\n", "================================================================================\n", "Running eval iteration 5...\n" @@ -199,15 +207,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 18 / 50 (36.0): 100%|██████████| 50/50 [00:02<00:00, 21.89it/s]\n" + "Average Metric: 5 / 50 (10.0): 100%|██████████| 50/50 [00:00<00:00, 1055.56it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 18 / 50 (36.0%)\n", - "Generating new signature for base...\n", + "Average Metric: 5 / 50 (10.0%)\n", "\n", "================================================================================\n", "Running eval iteration 6...\n" @@ -217,15 +224,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 6 / 50 (12.0): 100%|██████████| 50/50 [00:03<00:00, 13.65it/s]\n" + "Average Metric: 12 / 50 (24.0): 100%|██████████| 50/50 [00:00<00:00, 942.15it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 6 / 50 (12.0%)\n", + "Average Metric: 12 / 50 (24.0%)\n", "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 7 to avoid.\n", "\n", "================================================================================\n", "Running eval iteration 7...\n" @@ -235,69 +243,770 @@ "name": "stderr", "output_type": "stream", "text": [ - "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:02<00:00, 19.56it/s]" + "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:00<00:00, 1054.12it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Average Metric: 17 / 50 (34.0%)\n" + "Average Metric: 17 / 50 (34.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 8 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 8...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\n" + "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:00<00:00, 957.29it/s]\n" ] - } - ], - "source": [ - "from dspy.evaluate import Evaluate\n", - "from dspy.evaluate.metrics import answer_exact_match\n", - "from dspy.functional import TypedPredictor\n", - "from dspy.teleprompt.signature_opt_typed import optimize_signature\n", - "\n", - "evaluator = Evaluate(devset=devset, metric=answer_exact_match, num_threads=10, display_progress=True)\n", - "\n", - "program = optimize_signature(\n", - " student=TypedPredictor(BasicQA),\n", - " evaluator=evaluator,\n", - " initial_prompts=4,\n", - " n_iterations=8,\n", - " verbose=True,\n", - " prompt_model=gpt4,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ + }, { "name": "stdout", "output_type": "stream", "text": [ - "StringSignature(question -> answer\n", - " instructions='You are highly intelligent. Please provide short, factual answers to the following questions.'\n", - " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Inquiry:', 'desc': '${question}'})\n", - " answer = Field(annotation=str required=True json_schema_extra={'desc': 'usually between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Reply:'})\n", - ")\n" + "Average Metric: 17 / 50 (34.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 9 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 9...\n" ] - } - ], - "source": [ - "print(program.signature)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 12 / 50 (24.0): 100%|██████████| 50/50 [00:00<00:00, 1015.95it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 12 / 50 (24.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 10 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 10...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 11 / 50 (22.0): 100%|██████████| 50/50 [00:00<00:00, 839.64it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 11 / 50 (22.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 11 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 11...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:00<00:00, 833.32it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 12 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 12...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:00<00:00, 1105.97it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 13 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 13...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:00<00:00, 1112.59it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 14 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 14...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:00<00:00, 1096.58it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 15 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 15...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 1092.70it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 16 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 16...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 1097.79it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 17 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 17...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:00<00:00, 547.69it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 18 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 18...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 964.67it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 19 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 19...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 19 / 50 (38.0): 100%|██████████| 50/50 [00:00<00:00, 1014.22it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 19 / 50 (38.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 20 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 20...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 906.14it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 21 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 21...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:00<00:00, 1017.81it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 22 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 22...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 19 / 50 (38.0): 100%|██████████| 50/50 [00:00<00:00, 1032.48it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 19 / 50 (38.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 23 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 23...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:00<00:00, 726.33it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 24 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 24...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:00<00:00, 957.55it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 25 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 25...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:00<00:00, 1009.53it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 26 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 26...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0): 100%|██████████| 50/50 [00:00<00:00, 1064.53it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 15 / 50 (30.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 27 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 27...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 18 / 50 (36.0): 100%|██████████| 50/50 [00:00<00:00, 1052.90it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 18 / 50 (36.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 28 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 28...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 20 / 50 (40.0): 100%|██████████| 50/50 [00:00<00:00, 731.18it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 20 / 50 (40.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 29 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 29...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0): 100%|██████████| 50/50 [00:02<00:00, 18.61it/s]\n", + "/Users/ahle/repos/dspy/dspy/evaluate/evaluate.py:145: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n", + " df = df.applymap(truncate_cell)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 16 / 50 (32.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 30 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 30...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0): 100%|██████████| 50/50 [00:02<00:00, 18.23it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 50 (34.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 31 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 31...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 19 / 50 (38.0): 100%|██████████| 50/50 [00:02<00:00, 20.82it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average Metric: 19 / 50 (38.0%)\n", + "Generating new signature for base...\n", + "Tested the signature, and it's not in the list of 32 to avoid.\n", + "\n", + "================================================================================\n", + "Running eval iteration 32...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Average Metric: 17 / 49 (34.7): 98%|█████████▊| 49/50 [00:14<00:00, 20.66it/s]" + ] + } + ], + "source": [ + "from dspy.evaluate import Evaluate\n", + "from dspy.evaluate.metrics import answer_exact_match\n", + "from dspy.functional import TypedPredictor\n", + "from dspy.teleprompt.signature_opt_typed import optimize_signature\n", + "\n", + "evaluator = Evaluate(devset=devset, metric=answer_exact_match, num_threads=10, display_progress=True)\n", + "\n", + "result = optimize_signature(\n", + " student=TypedPredictor(BasicQA),\n", + " evaluator=evaluator,\n", + " initial_prompts=6,\n", + " n_iterations=100,\n", + " max_examples=30,\n", + " verbose=True,\n", + " prompt_model=gpt4,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check the final program after optimization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "predictor = Predict(BasicQA(question -> answer\n", + " instructions='Answer questions with short factoid answers.'\n", + " question = Field(annotation=str required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Question:', 'desc': '${question}'})\n", + " answer = Field(annotation=str required=True json_schema_extra={'desc': 'often between 1 and 5 words', '__dspy_field_type': 'output', 'prefix': 'Answer:'})\n", + "))" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result.program" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plot the scores over time" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABVjElEQVR4nO3deXhc5Xk3/u+ZVetIlmRrQZJ3MMYLYIMRNg6LA5iUQPDbXyBOYygXlNSkgK82iRuykISKpr8fIfR1TJuXmKbBcUtelkAbXDAgbPAqMIsJxjbGkrElW5al0Taj0cz5/THznDmznJlzZs6MlvP9XJcurJmR5vGx8bl13/dzP5IsyzKIiIiI8sQ22gsgIiIia2HwQURERHnF4IOIiIjyisEHERER5RWDDyIiIsorBh9ERESUVww+iIiIKK8YfBAREVFeOUZ7AfFCoRBOnDiB0tJSSJI02sshIiIiHWRZRl9fH+rq6mCzpc5tjLng48SJE2hoaBjtZRAREVEG2tvbUV9fn/I1Yy74KC0tBRBevMfjGeXVEBERkR5erxcNDQ3KfTyVMRd8iFKLx+Nh8EFERDTO6GmZYMMpERER5RWDDyIiIsorBh9ERESUVww+iIiIKK8YfBAREVFeMfggIiKivGLwQURERHnF4IOIiIjyisEHERER5RWDDyIiIsorBh9ERESUVww+iIiIKK/G3MFyRDS+dfX78dRbn2FgeCTm8QX1ZfjKRamP2SYia2DwQUSm2vTWUWx4/UjC45IEXDF7MqpK3KOwKiIaSxh8EJGpjpwaAABced5kXFDnAQA8ueMofIEQegYDDD6IiMEHEZnrWPcgAOAbTVNx9ZxqAMBz73yOE70+DMaVYojImthwSkSmkWUZbWfCmY/GimLl8UKXHQAwOBwclXUR0djC4IOITNM9MIyB4SAkCaifVKg8XuwOJ1mZ+SAigMEHEZlIlFxqPAUocNqVxwudzHwQURSDDyIyTduZcPDRWFEU87iS+fAz+CAiBh9EZKK2SOZjamVs8CF6PuJnfxCRNTH4ICLTHNPKfLDhlIhUGHwQkWnauiM7XSqLYx4vcrHhlIiiGHwQkWlE5mNqXOajiJkPIlJh8EFEphgaDuJUnx9AYs8HG06JSI3BBxGZov1sOOtRWuBAWaEz5jllq22AwQcRZRl8PPLII5AkCffff7/ymM/nw9q1a1FZWYmSkhKsWrUKnZ2d2a6TiMY4peRSWQRJkmKeK3ZHgg8/ez6IKIvgY+/evfiXf/kXLFiwIObxBx54AC+++CKeeeYZtLS04MSJE7jllluyXigRjW3KNtuK4oTnCiMNp9xqS0RAhsFHf38/Vq9ejV/96leYNGmS8nhvby+efPJJPProo7j66quxaNEibNq0CW+//TZ27dpl2qKJaOwRZ7o0xDWbAtGttkNsOCUiZHiq7dq1a/GlL30JK1aswE9/+lPl8dbWVgQCAaxYsUJ5bM6cOWhsbMTOnTtx2WWXJXwvv98Pv9+vfO71ejNZElFK77Sdxf/3PwfhD4RiHr98VhXWffHcvK8nGJLx4PMfYN45ZVi9ZGre3z8XjmkMGAPUQ8YYfBBRBsHHli1b8M4772Dv3r0Jz3V0dMDlcqG8vDzm8erqanR0dCT9fs3NzXjooYeMLoPIkF/vOIq3Dp9JeHzfsbP4fxbXo35S4g0zlw6c6MXv9rSjtOAkvnZpY0KPxHgULbsky3yE/6lh5oOIAINll/b2dtx33314+umnUVBQYMoC1q9fj97eXuWjvb3dlO9LpCaaIe+9ahae+PrFeOLrF2NOTSkAYMehrryv55Q3nO3r842gZzCQ9/c3WzAk43j3EACgMUnmQzScsueDiACDwUdraytOnTqFiy++GA6HAw6HAy0tLXj88cfhcDhQXV2N4eFh9PT0xHxdZ2cnampqkn5Pt9sNj8cT80FktmORfoQbF9bh+nm1uH5eLa69IPx3cvvh/Acfp/ujpUZRrhjPOrw+DAdDcNol1JYVJjxfqEw4ZeaDiAwGH9dccw0++OAD7N+/X/lYvHgxVq9erfza6XRi27ZtytccPHgQbW1taGpqMn3xRHr0DA7D6wv/xK0+c2T57CoAwFuHuxAMyXldU1efKviIBEbjmfg91E8qgt2WWEISDafDIyEEgqGE54nIWgz1fJSWlmLevHkxjxUXF6OyslJ5/M4778S6detQUVEBj8eDb33rW2hqakrabEqUD6IXYXKpW2l8BICFDeUocTvQMxjAgRO9WFBfnrc1dakyH+0TIPMhfg/xB8oJ6us+OBxEWSHnGxJZmen/Avz85z/Hn/3Zn2HVqlVYvnw5ampq8Oyzz5r9NkS6aZ034rTbcNmMSgDA9jz3fXT1Dyu/Fusbz7ROsxVcdhsckYwIm06JKKOttmpvvPFGzOcFBQXYsGEDNmzYkO23JjKFyHwka4Rcfm4VXv1TJ7YfOo21V83K25omWs9Hqm22ACBJEgpddvT5Rth0SkQ824UmvrYz2pM3l80K9320Hjub1+PerVZ2AbjdloiiGHzQhHesO9wM2ViZuAtjelUxzikvRCAoY/fR7rytSd1w2uH1wTfOD1yLnuuSGOAJRWK7Lc93IbI8Bh804bUp/QiJN0ZJknBFZNfL9k/y0/fhHwkqu28cNgmyDBw/O36zH72DAfQOhWeVNFQkBnhCkYsn2xJRGIMPmtD8I0Gc9PoAaPcjXDF7MgBgx+HTeVmTaDZ12iXMrg4POhvPTafq3URFLu02MvHcoJ/BB5HVMfigCe342SHIcvin7spiV9LXXD6zEpIEfNLZj45eX87XJEoulcVuTIsERG3juO9DlLWSjVVXK3JxyikRhTH4oAmtTbUFVOv8lEnFLsw/pwwAsCMP005Fs2lVqUtp0BzPmY9022wFNpwSkcDggyY0MXlTq+QiiL6PHYdyX3pRgo8St7L9dzxnPpQAL801LmTmg4giGHzQhNYWOews1S4MAFg2S/R9dCGU41HrouejqsStbP8d18FHmhkfghixzswHETH4oAmtLdKP0JCmJHDx1HIUuezo6h/Gxx19OV3T6UjPx+RSt1KqaOsezHnQkyttOmZ8ANHD5QbYcEpkeQw+KGt//OAklv3ja3i37ayhr/u7Z97DLb98K6cHjWmNVo/ndtixZHoFAGB7itLLkdP9WPFoC55953jGa1KXXerKC+CwSRgeCaGzL3fNrj97+WPctOEt9A4GdH/N0HAQX3p8O370hwOarxkeCeFEbzi7lGwrs5qS+Qiw7EJkdQw+KGv/9cFJHD87hG1/OqX7a/wjQfz+neN4p60HR7tyc6qrLMu6SwKAesutdtPp5t1tOHyqH/+5rz3jdUWDDxccdhvOmRSejZGrptOh4SD+z/ajeK+9B1s/6tD9dR983osDJ7z4j73tkOXkWZnjZweV3URVJcl3EwlFbmY+iCiMwQdlTdxM1SPD0xFbYAEoA6rMdqrPD/9ICHabhLpy7eFXgmg63XO0W3Pi6I7IAXRtWQQKStmlxA0AMaWXXNjzWTeGI9mlHQYO0BN/nkOBYMxZNGrHutPvJhKUIWNsOCWyPAYflDXRQGkk+FDfvI2UAowQmYS68gI47en/qs+aUoJqjxv+kRD2fpY4ar3T68PBznA/yEmvD/6RzH6CVxpOS+OCjxxlPrZ/Ei0jGWmoVf95aq2tTec2W0AdfDDzQWR1DD4oa+ImdVp1THw6YgsskLvMh7LNNk0vghAetR4pvSTJEKgfC49EHzK8puGRkPL7rYpkPkRJKFen26rLSN0Dw/jopFfX16nPn9EqCUXPdNETfETKLgw+iCyPwQdlZXgkhJ5I5kJ9s0pHbIEFgJ4cBR/ipNV0O13UlHNekgQf8Y2omWQqzgyEr5HdJqG80Akg2qiZi7LLKa8PH3f0QZKARVMnAUj+e0tGHUxqrU3Z6ZJmKzOg3mrLsguR1TH4oKyImykAnO73azYmxhNbYIEcZj4MNJsKS2eFg4+PTnqV3gwACIVk7Dh8BkA0Y6HO3ujV1Re+oVcWu2CzhXskomUX8xtvRdZjXl0ZblxQG3lM3yC1mLKLZvAROTFYR4CnDBljwymR5TH4oKyImykQzoL06TwuXZ3G9+as7KJvm61aVYkbc2s9AIC3j0QzBB939KGr349Cpx1/FrmJZ1ImETf0yZF+DyA6GfTsYABen7nXQpSKrphdhSvODZeU9n52Vtegr9MxZZfEwChmN5GOa1wc2e0yxFNtiSyPwQdlJb7JVE/pRX3TAoCeQf29Ika0d+sb+x0vWelFZAsum1GBWVNKYr6/EadVMz6EErdD2aZqZtOpLMvYHsl8LJtdhRlVxagrK8DwSAh7kjTUxovNfCT2t5zu88MXCO8mEtuFU1EOltMZoBLRxMXgg7ISvwWzS0fTqdgCK+Si7NLvH8GZgfBa9JQE1ETT6fZDp5UykghEls2eHG0QzSBQ6EoSfADRvhQz+z4OdvbhdF84W7No6iRIkoRlOs+wkWU5Jvjo6vcnBA0i86N3N5FoOPWPhBAcp9NcicgcDD4oKwmZDx3bbeNv2rloOBVlgopiF0oLnIa+dvG0SXA7bOj0+nH4VD98gSD2HA1nCq6YXZXVSHRRpqoqjR3INTUHp9tu/yQcMC2ZUQG3I5x1iAZWqZtOB4aD8AXCAWKhM/y18YGR3tNsBZH5ADjrg8jqGHxQVtQ9H4De4CMcGLgd4b9+uch8GJk/Ea/AacelkVHrbx7qwr7PzsI/EkK1x43ZU0pQV14Iu02CfySkOXxLi3j95LjMh9gtom7EzZYouYiAAwg31EpSuIflVIpx7qJ8Vui049zqcJkpPjASDbLpxqoLbocNkR5bzvogsjgGH5QVEWyI4ZZ6ej5Er8QFdeHGzlw0nOo97EzLFaryhNhiu2zWZEiSBKfdhnPKMxuJLq5PfNllqslll3C2Jrw7R/xegHAmaF5dGQDgrRRj5NWNsaIkFN/jYmR0PRCeo1IcKb0w+CCyNgYflBVxk5peFf7pV8+gMdErsKC+HADQMxjQvUVXr0y22aqJbMGuT7vx+sHwmTXLz43exBuVMomxTIVWz0djFn0kybQeOwtfIJqtURN9H6Isk3qdLtUQtNjf67EMArxCNp0SERh8UJbEdszzazwxn6cibrDzzwn/BD4Skk3/STibsgsAzKkpRVWJG0OBID7p7AcQnQECRIMFoztekm21BaKZjxM9Qxgeyf6UX6VBNpKtUbsi8vvYfrhLM+gTQWRViVuZEJtYdjF+jbndlogABh+UJXEznVNTGvN5KuKGfV5NKVz23PR9REsC+voR4kmShGWzKpXPz6/1xGQrlAZRA8FHIBjC2UExWj224XRyqRsFThtCcjgAyZYoFalLLsKiaZNQ4LThdJ9fOasmnggiq0rdSQOtmN1EBrJLonmVmQ8ia2PwQRlT30znRAZzpQs+4m9ansiI8R4TD5cLBEP4PHIDzzTzAcQ2ai6Pu4lnst22O/L7ttskTCqKDT4kSYqWctIENL1DqctUZ/r9OHAifH6LOlsjuB12LJkeDqy0TrlVl4fEuo6fHcJI5HRckfWYVOSEx8BuomI3D5cjIgYflAX1zVQM3upKM2Jd9EiIm1ZZYTgNb2bm42SPD8GQDLfDhilx5Q0jlqkCjmVxwUcmczlENqFCNVpdTewa+axLu4/khf2fY+FD/4Mte9s1X/PWkXCj6fm1noTyjqA01Go0nYrG2MklLtR4CuBy2DASknGyN7xDRhmrbjCzVMSGUyICgw/KgvpmKm7yvkAo5amlSp9A5KZVHskAmBl8dHjDN8jasoKkN3m9qj0FuP3yaVhxfrWSKRBEOad7YBh9OkeiazWbCvPOCWePUk0f/X3r8Zj/JrP9E+2Si3BhQzkA4FCknyXVWm02CQ2TYnf3ZDK6HojO+uCcDyJrY/BBGVPfoIrdDuXGkmq7bfxZIGWRskvvkHkj1tPd5I340ZcvwP9ZsxguR+z/KiVuByqLIyPRdWY/lD6KuH4PQQQLbx3uSjoBVD3sbH97T9JzYGRZVrIZqYIP0adxsjd5g6uYVFsVCSob4zI9RrfZCsx8EBHA4IOyIG5QIrUvbvap+j7it2dGgw/zMh9mBh+pKKUXnX0fyvXSWNfC+nKUuh3oGQzgwInehOf3ftatjKUPhmTsjJRX1I6c7sfJXh9cDhsumVahuZbJJW4UuewIycDxs4nr74obhiYyPWK7rQg+GjLNfLDhlMjSGHxQxuJ/khf/TbXdNlp2yWHwoezUSJ5hMIv4qV9v5kNrm63gsNvQNDNc3kk2/jy+OTRZs6j4uiXTK1DgtCc8L6gbXOPXPzg8omQmEjIf2ZZd2HBKRDAYfGzcuBELFiyAx+OBx+NBU1MT/vjHPyrPX3nllZAkKebjnnvuMX3RNDbE/3SsJ/OhVXYxc7dLspNjc8Hodls9GZnoibqJB7+9GQksblxYp/ma6HwP7ZKLoNU0K0bmFzhtKI5kKtSB1ohqN5HRrcxFznDZJVVfEBFNfIaCj/r6ejzyyCNobW3Fvn37cPXVV+Omm27CgQMHlNfcddddOHnypPLxs5/9zPRF09gQfzMVPyVrTTkNJLlp5SLzcVoc3jbmyi7pMzJie2/rsbMxTZmn+/z408nw9tm/vfZc2G0SPjszGDN7Y3gkhF2fnon5PqloHWZ3uj/csFtV4lYGlKkzHyciu4lcGewmim61ZdmFyMoMBR833ngjbrjhBsyePRvnnnsuHn74YZSUlGDXrl3Ka4qKilBTU6N8eDwe0xdNY0P8zTRd5uNEz1DCTau8aPz2fExVDoPTGXzoCIqmVhahflIhAkEZuz+N7noR57BcUOfB1MpiXNxYDiC2PPNO21kMDgdRVeJShr6lXr9G8JFknSLQ6vOPYP/xHgDhgMTobiI2nBIRkEXPRzAYxJYtWzAwMICmpibl8aeffhpVVVWYN28e1q9fj8HB1P8w+/1+eL3emA8aH+JvpqKXQWu3i/oIdnHTymXDqVZvhVnEzfvzniEEgulHousJiiRJUpVeooGFUk6JPLdsVjizseNwtPQiekCWzqrSFRSI7c7xI+KTrbPAaUeNpyDyPuH3NNrvAXCrLRGFGQ4+PvjgA5SUlMDtduOee+7Bc889h7lz5wIAvva1r+G3v/0tXn/9daxfvx7//u//jq9//espv19zczPKysqUj4aGhsx+J5R38TepyZGGU63MR3y/B2B+8CHLckIvSq5MKXXD7bAhGJLTjkQfCYbQPaivHCRKJqKnI7x9NjK7IxJ0LFO25Z5RtuVuV7bYpi+5ALHbZ9WD4aLBmyvp60UgZGSsuhANPpj5ILIyh9EvOO+887B//3709vbi97//PdasWYOWlhbMnTsXd999t/K6+fPno7a2Ftdccw2OHDmCmTNnJv1+69evx7p165TPvV4vA5BxINnNNFp2Sd7zkWx7ptlll4HhIHyBcBYi17tdxI6RQ6f6cezMYMrmy+6BYcgyYJPCQ9lSuXxmJSQJOHSqHx29Pnh9AXR6/XA7bFg8bRIAYGF9GUoLHOgdCuCDz3sxrbII70fKIXqaTQHgnPJC2KTwIW+n+/yYEslspDp5d89n3cqU00xG1ytlFz+DDyIrM5z5cLlcmDVrFhYtWoTm5mYsXLgQv/jFL5K+dsmSJQCAw4cPa34/t9ut7J4RHzT2dQ8m3kzT9XyI0erqwVQeVeYjlGSwllGi5FPksis3ulzSu91W7MCpKHbBnqYkUl7kwoL6cgDh8eci03Cpavusw27D5TPF+Syn8faRM5Bl4NzqEtSUFehau8thQ115YcL6RTlN6+Rd5fNMMh+i4TTAsguRlWU95yMUCsHvT36z2b9/PwCgtrY227ehMSY6Wt2t3EzFbpfB4WDSU0vbusVOl8SyiyyHmxmzXleemk0FcR5LuuCjq9/YDhzl2PtDp5Xyy/K4cooor7x5qEt5jegF0asxyY6XVJmP2K81fmJwdMgYMx9EVmboR8P169dj5cqVaGxsRF9fHzZv3ow33ngDW7duxZEjR7B582bccMMNqKysxPvvv48HHngAy5cvx4IFC3K1fhol0ZtptIRQ7LKjwGmDLxBCV78fxe7oXy9ZltEWyXyo0/Vuhx2FTjuGAkH0DgaUYCTjdaUZYW62xgpx5on2YXCA6qA2nU2wy2ZX4X+/fhg7DnUp/RHxh9uJxtR3jp1V3j/VSPVkplYW4e0jZ2JmlWgGH6o/N0kC6iPnvRhR7BJzPpj5ILIyQ8HHqVOn8I1vfAMnT55EWVkZFixYgK1bt+KLX/wi2tvb8eqrr+Kxxx7DwMAAGhoasGrVKjz44IO5WjuNomQ3U0mSUFXixvGzQ+jq98f0QJwZGMbAcDBy04r9Cbqs0BkOPkzo+8jXNlshut02dcOp0XVd3DgJRS47zgxEMybx22enVhajsaIIbd2D6PT64bRLWDJDe6R6MkrmRhU8aZ1Bo/7zrPEUpJygqqUwkvnwBUIIhuS0JSgimpgMBR9PPvmk5nMNDQ1oaWnJekE0PmjdTCeXhoMPMStCEGn9ZDetskInOrw+U4KP03EHouWaKEW0nRmALMvKUK540eulLyPjcthw2YxKvPbxKQDhjEay771sdhU2724DACyaOslwn0t8z8rQcFCZPhp/DScVOVHqdqDPP5JRsykQzXwA4UbXEnfu+3KIaOzh//lxPunsgwRgdnX6IU1G7f70jDLhU6gsceMKnXMZjGg7M4h9x2KPZrfbJFx57hSUFekvbfQOBfBJZ1/CIWVaN1OtptP2FAeRifX0mHCybb622Qr1kwohSeFdNmcGhjUzG0Z7PoBwwKEOPpJZrgo+9G6xVYs/30VcP5fDhtK4wECSJDRUFOGjk96Mg48Cpw2SFO7xGRweYfBBZFH8P18lEAxh1S/fhgxg5/qrUVqQXf+B2kcnvPjqv+5K+tymOy7BVedNMe29AGD1k7vQnqQU8GcLavG/v3ax7u/z989+gP/64CT+9S8W4doLapTHtW6mWsHHkdP9AJIPpjJz1kf0ULn8BB9uhx21ngKc6PXh2JlBzeAiWsowFnwIWttnm2ZWwSYBIdl4vwcQzdx09Q+j3z+iNOxOVo1WV5teVYyPTnoxrcp4sykQDmCKnHYMDAfDTafmx/hENA4w+FDxBYLKjotdn3bji3OrTfve4idLT4EDFzaGZzX86aQXp/v8CRMms9U7GFACD5Gu9weC2H20G28cPI1AMASnPf1GJ/9IENs+7gQAbD3QGRd8aJRdNAaNiePfL4r83tVMDT6Um2d+Gk6B8A38RK8P7d2DWDQ18fcHAO2RY+uNNGnOmlKK9SvnoNBlV2ZwxCsrdOIfvjIfp/v8mH9OmeG1ewqcmFTkxNnBANq7B9M27P7VF2ag0GXH/1pUb/i9hCK3Ixx8cNAYkWUx+FAZCUbnTOw4dNrU4KM/EtRc2DgJv/nLSwEAf/fMe3im9Tj6fOZ2/otAp6rEjX+/MzxrJRiSseinr6BnMID32nuweFr6xsTWY2eVgV07Dp+O6Wk4rbF7o0oZsR4toXh9Abzb3gMg+U/n5SL4MOFk23xvtQWAqRXF2PVpd8IZKcJIMITPz4aDQaNTQf/qC8mH86ndemmjoe8Zr7GiCGcHe3HszCC6B5LP+BAW1Jfj//3z8qzejyPWiSjrOR8TyYhqyJX6XA0z9PvCN1Z1Hb2kIPzrfhPmW6gd604c5mW3SViqzI7Q93tTv67T68ehU/3K51qZD/H5aVXmY9eR8AjwaZVFyXs+TC275OdEWzURUIjrHu9Ejw8jkQP1qkv1DQDLp0Zlx85AXnYLFSnbbZn5ILIqBh8qQVXw8WnXQEJzaDbEP7TqBjsRiCQbyJWNZGeoALGDq/QQB5W5IiUaEYwEQ7LyE3L8CPNkPR870pw5UmbSiPUB/wiGAsl3auSSaL7UKp+JoCSTU2DzYapq0Fh+go9w5mOImQ8iy2LwoRJ/MukOnTdpPURpRWQ71L/uN7vscib5zhIxpOq9471pb/TdA8P48EQvAOAbTVMBRK9H98AwQnJ40FRFUWzwkexk2/gTWeOJzEdPlmUXceMscNpQ7DI+gyJTWkfTC1rB4FjRqNpuqzXjw0wi+BjglFMiy2LwoRKMO1vkTRNLL/3+8I1VnfkocYdvumaMFVcTN8H4szfqJxVhRlUxgiFZaQDV8tbhLsgyMKemFLdcHG4u3PVpN/wjQeUmX1HkgiOucVXctAaGgxgaDqK9exBHuwZgt0loipxFEs+sskv0NNbkOzVyRWQ+TvX5MZSklKAVDI4V6u22SuYjh5kjMetjMMDgg8iqGHyojMQFH28d7koISDIlshul+ch8dCcPPoBow6c4ol2LKLksm1WFOTWlqCpxYSgQxDvHelKm5kvcDrgd4b9WXf1+peRyYUM5PBpbl8sj2ZNsg4/To9DvAYTX74n8WYpdLWpaweBYIdb1+dkhdHjDJ9bmo+wyaHLQTUTjB4MPlZFQuOxSXuREiduBnsEADkRKD9kSTaXJej7MbDgdHgnhRG+4VyXZT9rLIn0XO1JkdWRZVvpCrjh3Mmw2SZkzsf3QadVPx4mpeTFiHQg3naqDGC1mZz7yHXwA0dHjyUovqYLBsaC6tAAuhw0jIVnZop3T4EOcbMuGUyLLYvChIrbaFjjsuGxGuERg1q4X0fOhPmytOAfBx/Gzg5Dl8E+XyaZ8XjajAnabhM/ODGo2SH7aNYATvT647DZcGtmSqwQth7vS7igRKftTXh/eOhK+fsvPTR989PtHMBLXd2NEJoO8zBI9HTZ2x4ssy0rwkelU0Fyz2SQ0xM0fyeWEWLHbhVttiayLwYeKKLHYbZJys0yVITBCyXyoyy6R4MPMOR/HVDe6ZH0PpQVOXNxYDkA7sNr+STjrsXjaJOUgMFGu+eDzXnzS2QdA+wYlBny9cfA0egYDKHU7sLC+XHPNHtU18WZxLUZjwJjQGHdGitA9EJ4cmuxAvbFEfWicy26DpzB3I4CUhlNmPogsi8GHiii7OOzRMsO+Y92m/IQmgg/1nI9SZc5H9vMthFRnqAjLZoksRvK+j2RbY6s9BTi3ugSyDLx8oAOAdlOi2PHyXx+cBABcNrMyoTFVzWGPniPSM5j5+S75aJbUMrUiefAhPs/0FNh8UWdlqkpcOW3YjW61ZfBBZFUMPlRE2cVhkzC9qhjnlBciEJSx+2h3mq9MbyBF5sMXCGVVblBTmhtTBB9XRLI6bx0+k9BQGwiGlJ0w8dNIRTAiMjWaZZfI4+J1y3WcOeIxoe8jk8PbzBI93TZ58DFWd7oI6n6UXAdvypAxNpwSWRaDDxVxI3bYbJAkSbn5bv8k+9KLMucjSc8HYN7MAz07KxacU4bSAgd6hwL44PPYhtp323owMBxERbELc2s9Mc/Fz+nQmgURf/NfpuO01XITBo2NZsOpMmjs7GBMQKcnGBwLYjMfuQ4+IpkPbrUlsiwGHyoBVc8HEP1JP9221HSGR0Lwj4QzG6Xu6HZTl8OmbEvtM6n00haZppnqJ22H3YalM0VPS+zvTXy+dFZVwjTOJdMrlGmnQPrMBxA+SG2ajl0eZux46dI4byYfassK4bRLCARlZbsqMPa32QoxmY8c98ww80FEDD5UgpGeD6c9fNO9fGYlJAn4pLMfnaobilHqf2SL3bF1/1ITz3dR76xQNxAmI7IY8YPUxOdXJNkaW+RyxJzaqnWTV9+8xKm66WQbfAwOjygNjLm+eSZjt0lomJS440X04DSm+fMYbepm2FxnPoq51ZbI8iwTfJzu8+PHL36Ef3z5Y83XiJ4PkfmYVOzCgsgx5dlsuRWBRaHTntB4KcowZgwaO93nhy8Qgk0CzilPfXS7KCm9c+wsfvSHA3joxQP40R8O4P3jPQC0R6GrH68o1ii7qIIS0dyajlJ2STNiXZZl/HbXMXx0whvzuNj+63bYYkpb+SSyTeq+D/W5LmNZgdOOGk/40Lt8lV0YfBBZl2WCD68vgF+/dRRP7zqm+ZoRVc+HcFlkJPj+9rMZv3eyGR+CeMyMEetim21tWSFcjtR/tFMrizG9qhgjIRlPvf0ZNr31GZ56+zOEZODc6hLUaQQvV54XDibqJxXCqbGDpcZTALfDBpfdhqWzko9UjycaTnvSZD62HujEg89/iPu2vBvz+GlVv0c+R6urKWe8RP4cfIEgOr3hdY31ng8AmF1dAiD3zbGc80FEo/Mj4ihwRgKKVOPSleDDHr15TYqM/h4aznw3irLNtiDxcpuZ+Wgz2F/wz7ddhK0HOhCSo9fEJkm4YX6t5tdcUFeGX31jMao92j8dF7sd+PXtl8Buk5TR6enoLbu0fHIKAHDoVD8+7xlSMjyjuc1WaIzbbitKLqUFDiWzM5Y99OUL8PaRM7jqPH3Zqkwx80FElgk+7JGAIpAi+BA9H3ZVo6VosBzOYitsskPlBDN7Po4ZHOM975wyzIuUlYz44tzqtK9ZmmKcejLlheEgJdXJtuGx79Hy145Dp/HVSxoBjO6AMaExruwimk21Br6NNTMml2DG5JKcv0808xFEKCQnNDYT0cRnmbKLM/IPXKp5GgHVnA9BlC+GRzL/Ka0/so02WfAhHjOj87/tTPqdLmOVyHx4U2Q+jp0ZxPGzQ8rn6kAk3cj3fIie7xL+czAaDFqFyHwAgC+L/6+IaPyyTPAhGj1DMhDSyH4ocz5UvQzR4COLzIcvccCYIB4zY8S6crOrGNs7K5LRU3YRh91NipQw3jrcpfxZjuaMD0FkPry+EfQOBpRgsHEc/nnkUqFq0qtZ822IaHyxTPChLqWMaAQf0YbT6GvFHA4zyi6lSTMf0UPVstU+jn/SFj0RPUPa49VFpuP2y6ejxO3A2cEADkR2vShll1Hs+Sh02TEl8v7HugfG/Gm2o8VmkzhincjiLBN8OO3q4CN5ICFKMkl7PnKU+VB6PrLMfPT7R5Tx4o3j8GaXLvMxohr7ftWcydFThyMD4EbzRFu16Om2gzGH/FGs6OFy3PFCZEWWCT70ZD5E2cVpctlFbKNN1fORbeZDNDmWFznhKRj7Oyviia22vkAIviRjt9873oM+/wjKi5y4oK4sYfR9tOwyeg2nQDTw+6xrAMe7w/0pDD4ScbstkbVZJvhwqmZ3iGFi8UbixqsD0eDDb0LmI5dzPpQU/zi90ZW6HRCXPVnT6ZuRIGPprCrYbdFzd1qPncXQcDB6qNwoll2AaL/Nns+6MRwMwWGTNGemWBm32xJZm2WCD5tNUm5u6couDtO32uqZ85Hd2S7iTJexPsZbi80mpTzZdsfh2LHv4tTh4WAILZ+cVq7xqJddKsOBxp7IScj1kwpjglkKU8oubDglsiTLBB9AdHJpusyHesiYKbtdUpRdzJrzEZ0pMX5/ytbq+/D6Atjf3gMgOt5dferw8+9+DiAcKHqSBHj5JHa2iEzZeA0Gc02UXYYCLLsQWZG1gg+7mPWRZqutqkTjNKHhVGyjTT3nI7ufANvG8TZboVyMWI8bNLbzyBkEQzJmVBXHHIAmApHXPg5PPa0qcY36MK/4nS3jtQyWa8x8EFmbtYIPMWhMo+wSCCb2fJix1VYMEEs95yPbsos4PXX83uy0yi47Ilts4w+7WzqzCpIU/bMZ7X4PAKgsdsUM0WKzaXKi14lbbYmsyVDwsXHjRixYsAAejwcejwdNTU344x//qDzv8/mwdu1aVFZWoqSkBKtWrUJnZ6fpi86UGB6mvdsl0vORo7JLqTtxF0qpareLLGuPfk9lJBjC52fH/84KrbKLGC62LG5k+6RiF+arxsNPHuV+DyBcDlL/GYznYDCXCrnVlsjSDAUf9fX1eOSRR9Da2op9+/bh6quvxk033YQDBw4AAB544AG8+OKLeOaZZ9DS0oITJ07glltuycnCMyEyHwGNLEayIWMi+ND6Gj30TDgNycBQki2mepzo8WEkJMPlsCnHoo9H0UFj0eCjvXsQn50ZhN0moWlm4gm56oBktJtNBXXphQPGkivmbhciSzPUnXfjjTfGfP7www9j48aN2LVrF+rr6/Hkk09i8+bNuPrqqwEAmzZtwvnnn49du3bhsssuM2/VGRJBhdbJtiNK2UU158Mugg85o0OwQiEZ/cPaPR+FTjtsUjj46PeNKI14RoiSS8OkwnF9SFey813EVNOLGspRmmR+yRWzJ+OXbxwBAFSVju6MD2Gqqsl0PGeicqmQcz6ILC3jrQHBYBDPPPMMBgYG0NTUhNbWVgQCAaxYsUJ5zZw5c9DY2IidO3dqBh9+vx9+v1/53Ov1ZrqktByqQCIZkflwJsl8AOHeggKbPeHrUhkMBCGqKcm22kqShGK3A32+EfT5RzBFx/d89H8O4u3ItE8A6B6ITDYd5zc6EXy89P4JfPh5L4BoYBXf7yFcPLUchU47hgLBMZP5EAf7VZW4MwomrUBkPl7+sAMfn+wb5dUQWU9deSEev+2iUXt/w/8yfvDBB2hqaoLP50NJSQmee+45zJ07F/v374fL5UJ5eXnM66urq9HR0aH5/Zqbm/HQQw8ZXngmortdkpdQRM+HPUnPBxAJPpzGgg9RcrHbJKV5NV5pJPjQM2K9dyiAx187nPS5+fXlhtY21syMHOfe1T+sDA0TVpxfnfRr3A47rr2gGi/sP4EL6sqSvibfFkT6UBbUj431jEUiO5Tsz5qIcm/G4Oj+f2c4+DjvvPOwf/9+9Pb24ve//z3WrFmDlpaWjBewfv16rFu3Tvnc6/WioaEh4++Xit6yS7IhY0BmTafiULkSt0NzG2hJgQPo1TfroyfyF8btsOEXt16oPO522nF5kp6I8eTqOVPwn3/VhO4Bf8zjtWWFmHeO9o38H74yH3+1fCbm1nlyvURdFjaU4w/3Lh3X255z7dq51fiPuy/D2VH+B5DIqkY7K2v43V0uF2bNmgUAWLRoEfbu3Ytf/OIX+OpXv4rh4WH09PTEZD86OztRU1Oj+f3cbjfc7vyky8X8jkDaU22jAYckSXDZbRgOhjIKPlLN+BCMnO8iZmBUFLtw/bxaw+sZyyRJwqXTKwx/XbHbMWYCD2HBOM9C5ZrNJmHJjPEdLBNR5rKe8xEKheD3+7Fo0SI4nU5s27ZNee7gwYNoa2tDU1NTtm9jCnGybVBjzkcwyYRTILvttmKIUrJ+D6Ek0kipt+wCRPsjiIiIxhtDmY/169dj5cqVaGxsRF9fHzZv3ow33ngDW7duRVlZGe68806sW7cOFRUV8Hg8+Na3voWmpqYxsdMFiA4P02o4Fdtp48/icDlsgD+zQWPqsouWUgOZDxF8eBh8EBHROGUo+Dh16hS+8Y1v4OTJkygrK8OCBQuwdetWfPGLXwQA/PznP4fNZsOqVavg9/tx3XXX4Ze//GVOFp4JZchYmvHq6hNwAdXhctmUXVJlPoyUXSLBRzmDDyIiGqcMBR9PPvlkyucLCgqwYcMGbNiwIatF5Uq68eqi5yNp5gPRw8KMSHWonBAdsZ4++PCy7EJEROOctc52SZP5GEkyXh3IrudD9HGk6vkoVjIf6c93Yc8HERGNd5YKPpzpMh/BxN0ugKrsklHPRzj4KE6xrUnp+dCR+RBbbcUociIiovHGUsGHXQk+Um+11Sq7ZNTzkeJEW0E8Z6ThlJkPIiIarywVfDjTll0Sh4wB5pRdzJrzoQQfRWPjHBMiIiKjLBV8KOPVNTIfQa2eD6XsYvwEzgF/+p4PI5kPMWSMmQ8iIhqvLBV8KGUXjd4NzZ4PM8oubu1gwUjPB3e7EBHReGep4EPM7zDc85HFnI9+PXM+Muj54JwPIiIarywVfNiVU23TDBnT2mqr8XWp6Jrz4dY35yMQDGFgOFz6YeaDiIjGK0sFH+m22qYcr45MT7XV0fMRCT78I6kPrxNZD4Dj1YmIaPyyVPAhhoxpne0STHKqLWDObpfiFJkP9XMDKUovIvgoLXAkBEhERETjhbWCD1vqU21HtE61zXC3i38kqAwmS1V2cdptKHCG3yNV3wd3uhAR0URgreDDnvpUW7ELJn7OhzvDzId690qq4CP8fDigSBV8iJ0unG5KRETjmbWCj0g5JZinCacikChy2dOWSUp17HjhdFMiIpoILBZ8pG44je52MedsFz07XYQSHbM+xLkuDD6IiGg8s1bwkabhVGzB1cp8+DMsu6Sa8SEo221TZj7Cz5UVcrQ6ERGNX9YKPpSGU62yi8Z49SzLLqV6Mh8F6TMfLLsQEdFEYK3gQ2k4TQwiQiEZIiYxa6ttv44TbQVlxLo/oPmaniGWXYiIaPyzWPChfaqteuS65nh1gz0fYmJpsSt98FGso+eDu12IiGgisFbwYdM+1VZditEcr57DzId4TeqeD5ZdiIho/LNo8JEYRKgfi898ZDvnQ1fPR+Q1qSaccsgYERFNBJYKPpypyi6qx+J7PpzZbrXV0/PBOR9ERGQRlgo+7CkzH+HgQ5LMHzImppemoudk2x4GH0RENAFYKvgQvRzJMh/RQ+USJ5EqDad5mPOhlfnwBYLK+7PhlIiIxjNLBR+inJKs4VRsv002Bl3JfGRYdjFjzocoudhtkq6JqURERGOVpYIPu1277KKMVrclXpJMyy59Bsarl6Y5WE40m3oKHJCk1OfEEBERjWWWCj5EYJFqzofdnnhjd2ea+fCFA4ZiHcFHsdse+ZrUmY/yIo5WJyKi8c1SwYc9xZwPZbR60p6PcGCQ8Xh1A3M++odHIMuJ6xPBh4fNpkRENM5ZKviINpwm2e0SFA2n5pVdlIZTA2UXWQYGh4MJz/NEWyIimigsFXykOtVW9HykajgdCckIaRxKFy8UkjEQCSL07HYpcNqU907W96GUXRh8EBHROGet4CPFqbZaJ9oC0eAD0N/3MTAcDSD0ZD4kSUo568PLGR9ERDRBWCv4SLHbJVp20Z7zAQB+naUXkb1w2iWlYTWdVLM+OGCMiIgmCkPBR3NzMy655BKUlpZiypQpuPnmm3Hw4MGY11x55ZWQJCnm45577jF10ZlKNecjOmQs8ZKoD5rT2/eh7vfQuzW2NMWsj16eaEtERBOEoeCjpaUFa9euxa5du/DKK68gEAjg2muvxcDAQMzr7rrrLpw8eVL5+NnPfmbqojOlHCyXpOcjkKLnQ5Ikw4PG+gyc6yJEMx+BhOe424WIiCYKQ6MyX3755ZjPn3rqKUyZMgWtra1Yvny58nhRURFqamrMWaGJRNklkCSACEZKMc4kPR8A4LbbMDwSyiDzoT9YKE7R8yGGjLHhlIiIxrusej56e3sBABUVFTGPP/3006iqqsK8efOwfv16DA4Oan4Pv98Pr9cb85EroqSStOE0qJ35AIxvt40eKmfXvb6SFCfbsuGUiIgmiowPCQmFQrj//vuxdOlSzJs3T3n8a1/7GqZOnYq6ujq8//77+M53voODBw/i2WefTfp9mpub8dBDD2W6DEOiDacyZFmO6cUYSdHzAWQQfBiY8SGIM2AGUmy1LWPPBxERjXMZBx9r167Fhx9+iB07dsQ8fvfddyu/nj9/Pmpra3HNNdfgyJEjmDlzZsL3Wb9+PdatW6d87vV60dDQkOmyUlKf2zISkmNKLErwoVF2ifZ8JA4ASyba86E/WFC22sYFH7IsK7tdygs5Xp2IiMa3jIKPe++9Fy+99BLefPNN1NfXp3ztkiVLAACHDx9OGny43W643e5MlmGY+tyWYEiGU1URET0fWmUXZ2S7rd6ttgMGDpUTtE62HRgOKqUill2IiGi8M9TzIcsy7r33Xjz33HN47bXXMH369LRfs3//fgBAbW1tRgs0k3qGR3zTaSDFnA8gOuvDaM+HnnNdBK05H6Lk4rLbUOC01GgWIiKagAxlPtauXYvNmzfjhRdeQGlpKTo6OgAAZWVlKCwsxJEjR7B582bccMMNqKysxPvvv48HHngAy5cvx4IFC3LyGzDCqRoWFt90qsz5sJvT89GXSc+HRuZDOdelyKl7ZggREdFYZSj42LhxI4DwIDG1TZs24fbbb4fL5cKrr76Kxx57DAMDA2hoaMCqVavw4IMPmrbgbKiTGvHnu0QbTlP3fCQ7FyaZ/kzKLpFtufE9H73c6UJERBOIoeAj2VHvag0NDWhpaclqQbkkSRKcdgmBoJwwYl2cdKvV8+E22HDa7wsHDEaGjBVHtuXGZz64zZaIiCYSyzUQ2DWmnIqyi1Or7JJhz0dGZRd/fNmFA8aIiGjisFzw4dQ432UkxXh1IPOej2JDwUc4uDg7OIyQan0suxAR0URiueBDGTQWTF52SdfzoXerrS8QLs8Uu/RPOJ1WWYwilx19vhF83NGnPM4BY0RENJFYLviwp8l8aA4Zsxs7WG4oEnwUOPUHHy6HDZfNqAQAbD90Wnm8h5kPIiKaQCwXfDjtqXs+zBqvPjQcDj4KDWQ+AGDZrCoAwI7DXcpjLLsQEdFEYrngI3q+S/IhY2b1fPgC4dcVGsh8AMDyc8PBx+6j3UrpRux2KWfZhYiIJgDrBR8aZRcxXj3t2S46go+RYEgpzxgNPmZOLkGNpwDDIyHs/awbQHS3CzMfREQ0EVgw+AgHF/Hj1dMNGXMb6PnwqQIUo2UXSZKwbHY4+7H9ULj0wrILERFNJJYLPkRZJX68+ohSdsm+50P0ewDR4WRGXKEZfPBEWyIiGv8sF3yIIWLxDaci8+E0oedD9GoUOu0ZncWyNNJ0+qeTXpzy+uD1MfNBREQTh+WCD9HTEV92ET0f9jRbbf06yi7RbbaZXd6qEjcuqPMAAF4+0AEx1Z7BBxERTQTWCz7SlF20ej6cGWY+MiX6Pl567yQAoMhlV7IvRERE45nl7mZit0tAa8iYVs+HgbNdRM9HgcFmU7UrZk0GAOyJ7Hhh1oOIiCYK6wUfGuPVg+kmnBppODUh87F42qSYZlUGH0RENFFYL/gQp9rGZT5ED4jWkDERCMT3iiRjRtmlwGnHpdMrlM8ZfBAR0URhveBDY7dLUNntkmarrYGGU6MzPuItnz1Z+TWDDyIimiisF3woDafJh4xpjle3hwMJfT0f4dcYOVQuGdF0CnC0OhERTRzWCz7sonwS33Bq3nh1M3o+AGBOTSmqStwAmPkgIqKJw3LBh9OW/GC56Fbb1GUXf5622gLhUetfODdceqn2FGT1vYiIiMYKx2gvIN/sGg2nwbRlFwM9H8Pm9HwAwHeuPw/zzvFg1aL6rL8XERHRWGC54EOr4TSQ5mC5TMou2fZ8AMAUTwHuWDo96+9DREQ0Vliv7KI55yN1z4d7FHo+iIiIJiLLBR9aZRe9PR96yi6+4ezOdiEiIprILHd3VE611Rivnq7nIxiSE86FiecbMa/ng4iIaKKxXPChTDjVGjKWZqstkL70opztwrILERFRAusGHwlDxlKPVzcUfLDng4iISJP1gg+tIWNpej4cNglSJC7xB4Mp32MoEA5OGHwQERElslzwYU8zXl1rt4skSdFZH2kyHz4T53wQERFNNJYLPqJbbZP3fGjN+QCgO/gwc84HERHRRGO54EOUVQJxO1YCwdQ9H4D+7bbs+SAiItJmveDDnrzsEt3ton1J9E45ZdmFiIhIm6Hgo7m5GZdccglKS0sxZcoU3HzzzTh48GDMa3w+H9auXYvKykqUlJRg1apV6OzsNHXR2VAyHwmn2qae8wFEg48AMx9EREQZMxR8tLS0YO3atdi1axdeeeUVBAIBXHvttRgYGFBe88ADD+DFF1/EM888g5aWFpw4cQK33HKL6QvPVDTzEb/bJTJeXUfPR6qTbQPBkBLIMPggIiJKZOhguZdffjnm86eeegpTpkxBa2srli9fjt7eXjz55JPYvHkzrr76agDApk2bcP7552PXrl247LLLzFt5hkRwoc5ehEIyRCziyLLsIrIeAFDgslxVi4iIKK2s7o69vb0AgIqKCgBAa2srAoEAVqxYobxmzpw5aGxsxM6dO5N+D7/fD6/XG/ORS8lOtVWPWtfVcJoi+BD9HpIUzZQQERFRVMZ3x1AohPvvvx9Lly7FvHnzAAAdHR1wuVwoLy+PeW11dTU6OjqSfp/m5maUlZUpHw0NDZkuSReHLbHsov61rq22KXo+fKoBY5Kk/b2IiIisKuPgY+3atfjwww+xZcuWrBawfv169Pb2Kh/t7e1Zfb90lLKLareLetS61pAxwFjZhf0eREREyRnq+RDuvfdevPTSS3jzzTdRX1+vPF5TU4Ph4WH09PTEZD86OztRU1OT9Hu53W643e5MlpERZ7KyS1Cd+dCOx9wGgg8OGCMiIkrOUOZDlmXce++9eO655/Daa69h+vTpMc8vWrQITqcT27ZtUx47ePAg2tra0NTUZM6Ks2RXDpZL3vORouqia8jYEGd8EBERpWQo87F27Vps3rwZL7zwAkpLS5U+jrKyMhQWFqKsrAx33nkn1q1bh4qKCng8HnzrW99CU1PTmNjpAkTLKiOqACI6YExK2aehZ7y6j2UXIiKilAwFHxs3bgQAXHnllTGPb9q0CbfffjsA4Oc//zlsNhtWrVoFv9+P6667Dr/85S9NWawZlLKLKtuhZ7Q6EM18pJrzwZ4PIiKi1AwFH7Isp31NQUEBNmzYgA0bNmS8qFyKll0SMx+p+j0AnQ2nkbJLAcsuRERESVluEIXTpj3nI9VOFwBw2cMBRcqeDyXzYblLS0REpIvl7pDJG07Tj1YHdA4ZY9mFiIgoJcsFH84kDaciC5K25yPytXrKLtztQkRElJzlgo9k49VN7fngnA8iIqKUrBd8pCq7pOv50DPng8EHERFRStYLPuyJu130l13Y80FERJQt6wUfkdJKICgrW4eVIWNpyy7pd7uoD5YjIiKiRBYMPqLZDVF5CYR0Zj4454OIiChr1gs+VH0dYrJp0GjPByecEhERZcx6wYeqtCLKLaLnI+2cD7v+hlMGH0RERMlZL/hQZTdE0DGic6ut28iQMZflLi0REZEulrtDqrMbgUi5ZSQXPR/MfBARESVlueBDkiQlyBBlF8M9Hyy7EBERZcxywQcQzX6IhtOA0Z4PXWUXBh9ERETJWDL4cMaNWA8qZRd949X9es52YeaDiIgoKUsGH/En24r/OnVvtQ0mfV6WZY5XJyIiSsOSwYczbsS6OOFW93h1jZ6P4WBIGVzG4IOIiCg5SwYfSuYjruwiyjFa0m219Q1HH2fZhYiIKDlLBh9inkd82UXvVtuQHM2WqPki5Ri7TUpbwiEiIrIqSwYfStklGFt2SbfbRZ0ZSVZ6UTebShKDDyIiomQsGXxoNZzqnfMBJC+9sNmUiIgoPUsGH1pbbdONV3fYJIiERqrgg6PViYiItFnyLikyHGK8uhgylq7nQ5KklDtefJzxQURElJYlgw8xTCwYNDZeHUh9vgtHqxMREaVnyeDDaYub8xHSN14dUG23TdZwyp4PIiKitCwZfDjscQ2nQX3j1YHU57sou114rgsREZEmawYfttiGU2W8uo7MR6qyi49lFyIiorSsGXzYY0+1FT0fdpN6Plh2ISIi0mbN4COS4QjGlV309HwoJ9smHTIWfozBBxERkTaLBh/h33YgfshYtj0fLLsQERGlZc3gI268elDnhFNAZ88Hh4wRERFpMnyXfPPNN3HjjTeirq4OkiTh+eefj3n+9ttvhyRJMR/XX3+9Wes1RXzZRfR+pBsyBgAuRzirwYZTIiKizBgOPgYGBrBw4UJs2LBB8zXXX389Tp48qXz87ne/y2qRZnNESieBuPHqTiNlF875ICIiyojD6BesXLkSK1euTPkat9uNmpqajBeVawmn2ob0jVcHVEPGOOeDiIgoIzlpTnjjjTcwZcoUnHfeefjmN7+JM2fOaL7W7/fD6/XGfORa4qm2HK9ORESUL6YHH9dffz1+85vfYNu2bfjHf/xHtLS0YOXKlQgGg0lf39zcjLKyMuWjoaHB7CUlUIaMifHqQf27XUTWJOnBcgw+iIiI0jJcdknn1ltvVX49f/58LFiwADNnzsQbb7yBa665JuH169evx7p165TPvV5vzgMQR1zmI2ig7KLM+Ug1ZIxlFyIiIk053xM6Y8YMVFVV4fDhw0mfd7vd8Hg8MR+5JhpORcYjYOBgOZdde7eL0vPBzAcREZGmnAcfx48fx5kzZ1BbW5vrt9ItvuE0aFLPhy8QfozBBxERkTbDZZf+/v6YLMbRo0exf/9+VFRUoKKiAg899BBWrVqFmpoaHDlyBN/+9rcxa9YsXHfddaYuPBsJDacGej5E8BHgVlsiIqKMGA4+9u3bh6uuukr5XPRrrFmzBhs3bsT777+Pf/u3f0NPTw/q6upw7bXX4ic/+Qncbrd5q86SM67sYvpWWwYfREREmgwHH1deeSVkWdZ8fuvWrVktKB9Eb0cgFDte3amn7KIxZEyWZVXDKcerExERabHkXdIef6ptyMh49eSZD/XuF2Y+iIiItFky+Egou2TQ8xG/1VbM+ADY80FERJSKJYOPaMNp7Hh1XbtdNMououTitEtKcENERESJLHmXjG61jR0ypmvOh1J2iZ3YKppNmfUgIiJKzZLBhyiviOFiYttsNj0fPNeFiIhIH2sGH3bRcBq/20V/z0d82UU514Wj1YmIiFKyZvAhMh+ZzPmwa2Q+hjndlIiISA9rBh9x49XFf431fCQvu7Dng4iIKDVrBh+qOR+hkIxI4kM5cC4V9nwQERFlx5rBhz1adgmqprUaajiN7/lQdrtY8pISERHpZsk7pTrzIbbbqh9PRTSlxg8ZG2LDKRERkS6WDj4CoZAyaAwwOGSMPR9EREQZsWbwoRqvLrbZAvrGq7tVZRf1AXs80ZaIiEgfawYfqrJLQFV20VF1UXo+ZBkxgYtvhMEHERGRHtYMPiLllUAwpBowJkGS9DecArFNp6LhlD0fREREqVky+FBOtQ3JSs+Hnp0uQLTnA4jt+2DPBxERkT6WDD6UU22DIWW3i55+DyDcLyLilNjggxNOiYiI9LBk8OG0qTMfkeBDx04XQZRe1Ntth1h2ISIi0sWSwYddGa8e3e2iZ8aHUOxyAADODAwrj/k44ZSIiEgXSwYfTlF2CYUQCBrr+QCAixrLAQA7j5xRHmPPBxERkT6WDD7EnI+QDCX40NvzAQBXzJ4MANh+6LTyGMsuRERE+lgy+FBnOXyRRlEjPR/LZlcBAPZ9dlYJOkTZpcBhyUtKRESkmyXvlE5VoCGGgxkpu8yoKkZdWQGGgyHs+awbAM92ISIi0suSwYe6xOKPBA1OA2UXSZKipZdPwqWXITacEhER6WLR4COx7GIk8wFESy87DncBiPZ8sOGUiIgoNUsGHzabpAwKE70aRno+AGDprCpIEvBxRx86vT5l5gfLLkRERKlZMvgAoqUXJfgwmPmoKHZhXl0ZAODVP3Uqj7PsQkRElJp1g49IpsM3YnyrrSBKL698FA0+WHYhIiJKzbrBRyTTITIfRns+AOCKSPDx9uHwsDGXw5bR9yEiIrIS6wYf9tjzWYz2fADAoqmTUOi0YzjIQ+WIiIj0sm7wEZf5MNrzAQBuhx1LZlQonzP4ICIiSs9w8PHmm2/ixhtvRF1dHSRJwvPPPx/zvCzL+MEPfoDa2loUFhZixYoVOHTokFnrNY3THttwas+g5wMAls2qUn7NnS5ERETpGb7jDgwMYOHChdiwYUPS53/2s5/h8ccfxxNPPIHdu3ejuLgY1113HXw+X9aLNZPozfBH5nw4Myi7ANFzXgA2mxIREenhMPoFK1euxMqVK5M+J8syHnvsMTz44IO46aabAAC/+c1vUF1djeeffx633nprdqs1UXS3S+YNpwBwbnUJppS6carPjwKnZatYREREupl6tzx69Cg6OjqwYsUK5bGysjIsWbIEO3fuTPo1fr8fXq835iMfxDh1MZk0k54PIDxqXWy5Zc8HERFReqYGHx0dHQCA6urqmMerq6uV5+I1NzejrKxM+WhoaDBzSZqUsouy2yXzS3HdBTUAgPpJhdkvjIiIaIIb9TrB+vXr0dvbq3y0t7fn5X1Fj0c2u12Ea+dW4/9+83L86MsXmLI2IiKiicxwz0cqNTXhDEBnZydqa2uVxzs7O3HhhRcm/Rq32w23223mMnSxK1ttMztYTk2SJCyaOsmUdREREU10pmY+pk+fjpqaGmzbtk15zOv1Yvfu3WhqajLzrbImyiyi4dSZRdmFiIiI9DOc+ejv78fhw4eVz48ePYr9+/ejoqICjY2NuP/++/HTn/4Us2fPxvTp0/H9738fdXV1uPnmm81cd9ZE2cVvQuaDiIiI9DMcfOzbtw9XXXWV8vm6desAAGvWrMFTTz2Fb3/72xgYGMDdd9+Nnp4eLFu2DC+//DIKCgrMW7UJ7FmeaktERESZMRx8XHnllZBlWfN5SZLw4x//GD/+8Y+zWliuOePHq2c4ZIyIiIiMsWyjQ3TImCi7WPZSEBER5ZVl77iOSLARDMmRz5n5ICIiygfrBh9xZRaWXYiIiPLDssFH/O4WZj6IiIjyw7LBhzOux4M9H0RERPlh2TtufJnFybILERFRXlg3+Igrs3DIGBERUX5YN/iIG6fOng8iIqL8sHDwEd9watlLQURElFeWvePGZzq41ZaIiCg/LBx8xO92YfBBRESUDxYOPlh2ISIiGg2WveOy4ZSIiGh0WDb4iJ/rYWfPBxERUV5YNviI7/GIn3hKREREuWHZO2582YUNp0RERPlh2eDDGZ/5YNmFiIgoLywbfMRnOpj5ICIiyg/LBh/OhN0ulr0UREREeWXZO258poMTTomIiPLDssFHfI8H53wQERHlh2WDD45XJyIiGh2WDT7ih4rF94AQERFRblj2jhs/VIyZDyIiovywbPAR32DKng8iIqL8sG7wkbDbxbKXgoiIKK8se8fleHUiIqLRYd3gIz7zweCDiIgoL6wbfMT3fHDIGBERUV5YN/iwcbw6ERHRaLDsHTe+zMKqCxERUX6YHnz86Ec/giRJMR9z5swx+22ypi6zOO3hdRIREVHuOXLxTS+44AK8+uqr0Tdx5ORtsqKeaMqdLkRERPmTk6jA4XCgpqYmF9/aNOqAg/0eRERE+ZOTu+6hQ4dQV1eHGTNmYPXq1Whra9N8rd/vh9frjfnIB/V4de50ISIiyh/Tg48lS5bgqaeewssvv4yNGzfi6NGjuOKKK9DX15f09c3NzSgrK1M+GhoazF5SUuqAgzM+iIiI8keSZVnO5Rv09PRg6tSpePTRR3HnnXcmPO/3++H3+5XPvV4vGhoa0NvbC4/Hk7N1+QJBzPn+ywCAao8bu/9+Rc7ei4iIaKLzer0oKyvTdf/OeSdoeXk5zj33XBw+fDjp8263G263O9fLSKBuOGXPBxERUf7k/K7b39+PI0eOoLa2NtdvZYi60sKeDyIiovwxPfj427/9W7S0tOCzzz7D22+/ja985Suw2+247bbbzH6rrEiSBGck6OBWWyIiovwxvexy/Phx3HbbbThz5gwmT56MZcuWYdeuXZg8ebLZb5U1h82GQDAYs/OFiIiIcsv04GPLli1mf8ucEbtcmPkgIiLKH0v/yC96PdjzQURElD8WDz7Cv33O+SAiIsofawcfkaCDW22JiIjyx9J3XQd3uxAREeWdtYOPSMaDPR9ERET5Y/HgQ4r5LxEREeWetYOPSMOpnT0fREREeWPpu67IeDhZdiEiIsobawcfbDglIiLKO0sHH2KsOns+iIiI8sfSwYfIeIjeDyIiIso9S991lfHqzHwQERHljbWDDx4sR0RElHfWDj4i5RYnyy5ERER5Y+m7rpO7XYiIiPLO0sGHnbtdiIiI8s7SwYdT2e3C4IOIiChfLB18RIeMWfoyEBER5ZWl77rlRS4AQFmhc5RXQkREZB2O0V7AaLrrihmon1SImy86Z7SXQkREZBmWDj4ml7rxjaZpo70MIiIiS7F02YWIiIjyj8EHERER5RWDDyIiIsorBh9ERESUVww+iIiIKK8YfBAREVFeMfggIiKivGLwQURERHnF4IOIiIjyisEHERER5VXOgo8NGzZg2rRpKCgowJIlS7Bnz55cvRURERGNIzkJPv7jP/4D69atww9/+EO88847WLhwIa677jqcOnUqF29HRERE40hOgo9HH30Ud911F+644w7MnTsXTzzxBIqKivDrX/86F29HRERE44jpp9oODw+jtbUV69evVx6z2WxYsWIFdu7cmfB6v98Pv9+vfN7b2wsA8Hq9Zi+NiIiIckTct2VZTvta04OPrq4uBINBVFdXxzxeXV2Njz/+OOH1zc3NeOihhxIeb2hoMHtpRERElGN9fX0oKytL+RrTgw+j1q9fj3Xr1imfh0IhdHd3o7KyEpIkmfpeXq8XDQ0NaG9vh8fjMfV7Uyxe6/zhtc4fXuv84bXOH7OutSzL6OvrQ11dXdrXmh58VFVVwW63o7OzM+bxzs5O1NTUJLze7XbD7XbHPFZeXm72smJ4PB7+Zc4TXuv84bXOH17r/OG1zh8zrnW6jIdgesOpy+XCokWLsG3bNuWxUCiEbdu2oampyey3IyIionEmJ2WXdevWYc2aNVi8eDEuvfRSPPbYYxgYGMAdd9yRi7cjIiKicSQnwcdXv/pVnD59Gj/4wQ/Q0dGBCy+8EC+//HJCE2q+ud1u/PCHP0wo85D5eK3zh9c6f3it84fXOn9G41pLsp49MUREREQm4dkuRERElFcMPoiIiCivGHwQERFRXjH4ICIioryyTPCxYcMGTJs2DQUFBViyZAn27Nkz2ksa95qbm3HJJZegtLQUU6ZMwc0334yDBw/GvMbn82Ht2rWorKxESUkJVq1alTCAjox75JFHIEkS7r//fuUxXmvzfP755/j617+OyspKFBYWYv78+di3b5/yvCzL+MEPfoDa2loUFhZixYoVOHTo0CiueHwKBoP4/ve/j+nTp6OwsBAzZ87ET37yk5izQXitM/fmm2/ixhtvRF1dHSRJwvPPPx/zvJ5r293djdWrV8Pj8aC8vBx33nkn+vv7s1+cbAFbtmyRXS6X/Otf/1o+cOCAfNddd8nl5eVyZ2fnaC9tXLvuuuvkTZs2yR9++KG8f/9++YYbbpAbGxvl/v5+5TX33HOP3NDQIG/btk3et2+ffNlll8mXX375KK56/NuzZ488bdo0ecGCBfJ9992nPM5rbY7u7m556tSp8u233y7v3r1b/vTTT+WtW7fKhw8fVl7zyCOPyGVlZfLzzz8vv/fee/KXv/xlefr06fLQ0NAornz8efjhh+XKykr5pZdeko8ePSo/88wzcklJifyLX/xCeQ2vdeb++7//W/7e974nP/vsszIA+bnnnot5Xs+1vf766+WFCxfKu3btkrdv3y7PmjVLvu2227JemyWCj0svvVReu3at8nkwGJTr6urk5ubmUVzVxHPq1CkZgNzS0iLLsiz39PTITqdTfuaZZ5TX/OlPf5IByDt37hytZY5rfX198uzZs+VXXnlF/sIXvqAEH7zW5vnOd74jL1u2TPP5UCgk19TUyP/0T/+kPNbT0yO73W75d7/7XT6WOGF86Utfkv/yL/8y5rFbbrlFXr16tSzLvNZmig8+9Fzbjz76SAYg7927V3nNH//4R1mSJPnzzz/Paj0TvuwyPDyM1tZWrFixQnnMZrNhxYoV2Llz5yiubOLp7e0FAFRUVAAAWltbEQgEYq79nDlz0NjYyGufobVr1+JLX/pSzDUFeK3N9Ic//AGLFy/Gn//5n2PKlCm46KKL8Ktf/Up5/ujRo+jo6Ii51mVlZViyZAmvtUGXX345tm3bhk8++QQA8N5772HHjh1YuXIlAF7rXNJzbXfu3Iny8nIsXrxYec2KFStgs9mwe/furN5/1E+1zbWuri4Eg8GE6arV1dX4+OOPR2lVE08oFML999+PpUuXYt68eQCAjo4OuFyuhIMCq6ur0dHRMQqrHN+2bNmCd955B3v37k14jtfaPJ9++ik2btyIdevW4e///u+xd+9e/M3f/A1cLhfWrFmjXM9k/6bwWhvz3e9+F16vF3PmzIHdbkcwGMTDDz+M1atXAwCvdQ7pubYdHR2YMmVKzPMOhwMVFRVZX/8JH3xQfqxduxYffvghduzYMdpLmZDa29tx33334ZVXXkFBQcFoL2dCC4VCWLx4Mf7hH/4BAHDRRRfhww8/xBNPPIE1a9aM8uomlv/8z//E008/jc2bN+OCCy7A/v37cf/996Ouro7XeoKb8GWXqqoq2O32hK7/zs5O1NTUjNKqJpZ7770XL730El5//XXU19crj9fU1GB4eBg9PT0xr+e1N661tRWnTp3CxRdfDIfDAYfDgZaWFjz++ONwOByorq7mtTZJbW0t5s6dG/PY+eefj7a2NgBQrif/Tcne3/3d3+G73/0ubr31VsyfPx9/8Rd/gQceeADNzc0AeK1zSc+1rampwalTp2KeHxkZQXd3d9bXf8IHHy6XC4sWLcK2bduUx0KhELZt24ampqZRXNn4J8sy7r33Xjz33HN47bXXMH369JjnFy1aBKfTGXPtDx48iLa2Nl57g6655hp88MEH2L9/v/KxePFirF69Wvk1r7U5li5dmrBl/JNPPsHUqVMBANOnT0dNTU3MtfZ6vdi9ezevtUGDg4Ow2WJvQ3a7HaFQCACvdS7pubZNTU3o6elBa2ur8prXXnsNoVAIS5YsyW4BWbWrjhNbtmyR3W63/NRTT8kfffSRfPfdd8vl5eVyR0fHaC9tXPvmN78pl5WVyW+88YZ88uRJ5WNwcFB5zT333CM3NjbKr732mrxv3z65qalJbmpqGsVVTxzq3S6yzGttlj179sgOh0N++OGH5UOHDslPP/20XFRUJP/2t79VXvPII4/I5eXl8gsvvCC///778k033cTtnxlYs2aNfM455yhbbZ999lm5qqpK/va3v628htc6c319ffK7774rv/vuuzIA+dFHH5Xfffdd+dixY7Is67u2119/vXzRRRfJu3fvlnfs2CHPnj2bW22N+Od//me5sbFRdrlc8qWXXirv2rVrtJc07gFI+rFp0yblNUNDQ/Jf//Vfy5MmTZKLiorkr3zlK/LJkydHb9ETSHzwwWttnhdffFGeN2+e7Ha75Tlz5sj/+q//GvN8KBSSv//978vV1dWy2+2Wr7nmGvngwYOjtNrxy+v1yvfdd5/c2NgoFxQUyDNmzJC/973vyX6/X3kNr3XmXn/99aT/Rq9Zs0aWZX3X9syZM/Jtt90ml5SUyB6PR77jjjvkvr6+rNcmybJqlBwRERFRjk34ng8iIiIaWxh8EBERUV4x+CAiIqK8YvBBREREecXgg4iIiPKKwQcRERHlFYMPIiIiyisGH0RERJRXDD6IiIgorxh8EBERUV4x+CAiIqK8YvBBREREefX/A+/8197v1vmnAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.plot(result.scores)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "\n", + "You are an instruction optimizer for large language models.\n", + "\n", + "I will give some task instructions I've tried, along with their corresponding validation scores.\n", + "- The instructions are arranged in order based on their scores, where higher scores indicate better quality.\n", + "- Your task is to propose a new instruction that will lead a good language model to perform the task even better.\n", + "- Be creative, and think out of the box.\n", + "- Don't repeat instructions, descriptions and prefixes that have already been attempted.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Analysis: Consider what made the previous instructions good or bad.\n", + "Proposed Signature: A signature that will likely lead to a high score.. Respond with a single JSON object. JSON Schema: {\"properties\": {\"instructions\": {\"description\": \"The instructions for the task\", \"title\": \"Instructions\", \"type\": \"string\"}, \"question_prefix\": {\"description\": \"The prefix for question\", \"title\": \"Question Prefix\", \"type\": \"string\"}, \"question_desc\": {\"description\": \"The description for question\", \"title\": \"Question Desc\", \"type\": \"string\"}, \"answer_prefix\": {\"description\": \"The prefix for answer\", \"title\": \"Answer Prefix\", \"type\": \"string\"}, \"answer_desc\": {\"description\": \"The description for answer\", \"title\": \"Answer Desc\", \"type\": \"string\"}}, \"required\": [\"instructions\", \"question_prefix\", \"question_desc\", \"answer_prefix\", \"answer_desc\"], \"title\": \"SignatureInfo[BasicQA]\", \"type\": \"object\"}\n", + "Score: The expected score for the new signature. Don't write anything after this number. (Respond with a single float value)\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"You are an expert in your field. Respond to the inquiries with short, factual answers.\",\"question_prefix\":\"Inquiry:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Response:\",\"answer_desc\":\"typically a few words, factual\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with extensive knowledge, provide precise and factual responses to the questions. Keep your answers brief, typically within 1-5 words.\",\"question_prefix\":\"Interrogative:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Rejoinder:\",\"answer_desc\":\"a concise, factual answer\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast database of information, your task is to provide accurate and succinct answers to the following questions. Your responses should be factual and typically consist of 1-5 words.\",\"question_prefix\":\"Prompt:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Retort:\",\"answer_desc\":\"a short, factual answer\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with access to a vast knowledge base, your task is to provide accurate, factual answers to the questions posed. The questions will be about various topics, and your responses should be succinct, typically consisting of 1-5 words.\",\"question_prefix\":\"Inquiry:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Response:\",\"answer_desc\":\"a brief, factual answer\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast reservoir of information, your task is to provide precise, factual answers to the questions asked. Your responses should be succinct, typically consisting of 1-5 words, and cover a wide range of topics.\",\"question_prefix\":\"Enquiry:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Rejoinder:\",\"answer_desc\":\"a brief, factual answer\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge base, your task is to provide accurate, factual answers to a wide range of questions. Your responses should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge and understanding.\",\"question_prefix\":\"Inquiry:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Retort:\",\"answer_desc\":\"a brief, factual answer demonstrating accuracy and understanding\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge repository, your task is to provide precise, factual answers to the questions posed. Your responses should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy and precision are paramount.\",\"question_prefix\":\"Interrogative:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Rejoinder:\",\"answer_desc\":\"a brief, factual answer demonstrating precision and breadth of knowledge\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge repository, your task is to provide precise, factual responses to the inquiries posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount.\",\"question_prefix\":\"Interrogative Scrutiny:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Succinct Rejoinder:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge base, your task is to provide precise, factual answers to the questions posed. Your responses should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount.\",\"question_prefix\":\"Interrogative Inquiry:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Succinct Riposte:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge base, your task is to provide accurate, factual answers to the questions posed. Your responses should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a broad spectrum of topics. Accuracy, precision, and brevity are paramount. Consider the context of the question to provide the most relevant answer.\",\"question_prefix\":\"Interrogative Assessment:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Concise Counterpoint:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge base, your task is to provide accurate, factual answers to the questions posed. Your responses should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a broad spectrum of topics. Accuracy, precision, and brevity are paramount. Consider the context of the question to provide the most relevant and accurate answer.\",\"question_prefix\":\"Interrogative Analysis:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Concise Clarification:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge repository, your task is to provide precise, factual responses to the questions asked. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount. Consider the context and nuances of the question to provide the most relevant and accurate answer.\",\"question_prefix\":\"Interrogative Query:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Precise Response:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and contextual understanding\"}\n", + "Score: 34.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge base, your task is to provide accurate, factual answers to the questions posed. Your responses should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a broad spectrum of topics. Precision, brevity, and accuracy are paramount.\",\"question_prefix\":\"Interrogative Probe:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Concise Rebuttal:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 36.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge repository, your task is to provide precise, factual responses to the inquiries posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount.\",\"question_prefix\":\"Interrogative Examination:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Succinct Rejoinder:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 36.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge base, your task is to provide precise, factual answers to the questions posed. Your responses should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a broad spectrum of topics. Accuracy, precision, and brevity are paramount. Remember, your goal is to provide the most accurate and concise answer possible.\",\"question_prefix\":\"Interrogative Assessment:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Concise Counterpoint:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 36.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge base, your task is to provide precise, factual answers to the questions posed. Your responses should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a broad spectrum of topics. Accuracy, precision, and brevity are paramount. Additionally, consider the context of the question to provide the most relevant and accurate answer.\",\"question_prefix\":\"Contextual Inquiry:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Contextual Response:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and contextual understanding\"}\n", + "Score: 36.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with an extensive knowledge base, your task is to provide accurate, factual answers to the questions posed. Your responses should be succinct, typically consisting of 1-5 words, and demonstrate your wide-ranging knowledge across various topics. Accuracy, precision, and brevity are key. Consider the context of the question to provide the most relevant and accurate answer.\",\"question_prefix\":\"Interrogative Exploration:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Concise Clarification:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 36.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge repository, your task is to provide precise, factual responses to the questions posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount. Consider the context of the question to provide the most relevant and accurate answer.\",\"question_prefix\":\"Interrogative Exploration:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Precise Rejoinder:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and contextual understanding\"}\n", + "Score: 36.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge repository, your task is to provide precise, factual responses to the questions posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics.\",\"question_prefix\":\"Examination:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Rejoinder:\",\"answer_desc\":\"a brief, factual answer demonstrating precision and understanding\"}\n", + "Score: 38.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge repository, your task is to provide precise, factual responses to the questions posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy and precision are paramount.\",\"question_prefix\":\"Examination:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Rejoinder:\",\"answer_desc\":\"a brief, factual answer demonstrating precision and understanding\"}\n", + "Score: 38.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge repository, your task is to provide precise, factual responses to the questions posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount.\",\"question_prefix\":\"Interrogative Probe:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Succinct Retort:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 38.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge base, your task is to provide precise, factual responses to the questions posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount.\",\"question_prefix\":\"Interrogative Query:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Succinct Rebuttal:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 38.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge repository, your task is to provide precise, factual responses to the questions posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount. Consider the context of the question to provide the most relevant and accurate answer.\",\"question_prefix\":\"Interrogative Scrutiny:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Precise Rejoinder:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and contextual understanding\"}\n", + "Score: 38.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge repository, your task is to provide precise, factual responses to the questions posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount. Consider the context of the question to provide the most relevant and accurate answer.\",\"question_prefix\":\"Interrogative Scrutiny:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Precise Rejoinder:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and contextual understanding\"}\n", + "Score: 38.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a vast knowledge repository, your task is to provide precise, factual responses to the questions asked. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount. Consider the context and nuances of the question to provide the most relevant and accurate answer.\",\"question_prefix\":\"Interrogative Dissection:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Precise Elucidation:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and contextual understanding\"}\n", + "Score: 38.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge repository, your task is to provide precise, factual responses to the inquiries posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount.\",\"question_prefix\":\"Interrogative Examination:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Succinct Rejoinder:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 40.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge repository, your task is to provide precise, factual responses to the inquiries posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount.\",\"question_prefix\":\"Interrogative Examination:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Succinct Rejoinder:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 40.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge base, your task is to provide precise, factual answers to the questions posed. Your responses should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a broad spectrum of topics. Accuracy, precision, and brevity are paramount.\",\"question_prefix\":\"Interrogative Assessment:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Concise Counterpart:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and understanding\"}\n", + "Score: 40.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge repository, your task is to provide precise, factual responses to the questions posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount. Consider the context and nuances of the question to provide the most relevant and accurate answer.\",\"question_prefix\":\"Interrogative Dissection:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Precise Elucidation:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and contextual understanding\"}\n", + "Score: 42.0\n", + "\n", + "---\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge repository, your task is to provide precise, factual responses to the questions posed. Your answers should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a wide array of topics. Accuracy, precision, and brevity are paramount. Consider the context and nuances of the question to provide the most relevant and accurate answer.\",\"question_prefix\":\"Interrogative Dissection:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Precise Elucidation:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and contextual understanding\"}\n", + "Score: 42.0\n", + "\n", + "---\n", + "\n", + "Analysis:\u001b[32m The instructions that scored higher were those that emphasized the AI's extensive knowledge base and its ability to provide precise, factual responses. They also stressed the importance of brevity, typically limiting responses to 1-5 words. The instructions that scored lower did not emphasize these points as strongly. The use of unique prefixes for questions and answers, such as \"Interrogative Dissection\" and \"Precise Elucidation\", seemed to contribute to higher scores as well.\n", + "\n", + "Proposed Signature: {\"instructions\":\"As an AI with a comprehensive knowledge base, your task is to provide precise, factual answers to the questions posed. Your responses should be succinct, typically consisting of 1-5 words, and demonstrate your extensive knowledge across a broad spectrum of topics. Accuracy, precision, and brevity are paramount. Consider the context and nuances of the question to provide the most relevant and accurate answer.\",\"question_prefix\":\"Interrogative Probe:\",\"question_desc\":\"${question}\",\"answer_prefix\":\"Concise Clarification:\",\"answer_desc\":\"a brief, factual answer demonstrating precision, brevity, and contextual understanding\"}\n", + "\n", + "Score: 44.0\u001b[0m\n", + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "gpt4.inspect_history(n=1)" + ] } ], "metadata": { diff --git a/examples/quiz/DSPy_QuizGen_Cache b/examples/quiz/DSPy_QuizGen_Cache new file mode 160000 index 000000000..27d6d433e --- /dev/null +++ b/examples/quiz/DSPy_QuizGen_Cache @@ -0,0 +1 @@ +Subproject commit 27d6d433e73b91d3cf677ecf1d757813fcbd611d diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py index 2289ce697..2d680d3b9 100644 --- a/tests/functional/test_functional.py +++ b/tests/functional/test_functional.py @@ -635,3 +635,20 @@ class GenericSignature(dspy.Signature, Generic[T]): dspy.settings.configure(lm=lm) assert predictor().output == 23 + + +def test_field_validator_in_signature(): + class ValidatedSignature(dspy.Signature): + a: str = dspy.OutputField() + + @pydantic.field_validator("a") + @classmethod + def space_in_a(cls, a: str) -> str: + if not " " in a: + raise ValueError("a must contain a space") + return a + + with pytest.raises(pydantic.ValidationError): + _ = ValidatedSignature(a="no-space") + + _ = ValidatedSignature(a="with space") diff --git a/tests/functional/test_signature_opt_typed.py b/tests/functional/test_signature_opt_typed.py index 44adb9d0e..85c8c8d56 100644 --- a/tests/functional/test_signature_opt_typed.py +++ b/tests/functional/test_signature_opt_typed.py @@ -1,12 +1,10 @@ -import json +from typing import Generic, TypeVar + +import pydantic import dspy from dspy.evaluate import Evaluate from dspy.functional import TypedPredictor -from dspy.teleprompt.signature_opt_typed import ( - GenerateSignature, - make_info, - optimize_signature, -) +from dspy.teleprompt.signature_opt_typed import optimize_signature, make_info from dspy.utils import DummyLM from dspy.evaluate import Evaluate @@ -14,11 +12,6 @@ from dspy.functional import TypedPredictor -class BasicQA(dspy.Signature): - question: str = dspy.InputField() - answer: str = dspy.OutputField() - - hotpotqa = [ ex.with_inputs("question") for ex in [ @@ -106,44 +99,11 @@ class BasicQA(dspy.Signature): ] -def old_test_signature_info(): - info = make_info(BasicQA) - SignatureInfo = type(info) - - devset = [ - dspy.Example( - instructions="Answer the following questions", - question_desc="Some question to answer", - question_prefix="Q: ", - answer_desc="A short answer to the question", - answer_prefix="A: ", - ), - ] - - lm = DummyLM( - [ - json.dumps(dict(devset[0])), # Proposed signature - ] - ) - dspy.settings.configure(lm=lm) - - generator = TypedPredictor(GenerateInstructionGivenAttempts[SignatureInfo]) - - res = generator(attempted_signatures=[ScoredSignature[SignatureInfo](signature=info, score=50)]) - assert res.proposed_signature == SignatureInfo(**devset[0]) - - # Test the "to_signature" method - - class OutputSignature(dspy.Signature): - """Answer the following questions""" - - question: str = dspy.InputField(desc="Some question to answer", prefix="Q: ") - answer: str = dspy.OutputField(desc="A short answer to the question", prefix="A: ") - - assert res.proposed_signature.to_signature().equals(OutputSignature) - - def test_opt(): + class BasicQA(dspy.Signature): + question: str = dspy.InputField() + answer: str = dspy.OutputField() + qa_model = DummyLM([]) prompt_model = DummyLM( [ @@ -154,7 +114,7 @@ def test_opt(): ) dspy.settings.configure(lm=qa_model) - program = optimize_signature( + result = optimize_signature( student=TypedPredictor(BasicQA), evaluator=Evaluate(devset=hotpotqa, metric=answer_exact_match, num_threads=1), initial_prompts=1, @@ -172,4 +132,62 @@ class ExpectedSignature(dspy.Signature): question: str = dspy.InputField(desc="$q", prefix="Q:") answer: str = dspy.OutputField(desc="$a", prefix="A:") - assert program.signature.equals(ExpectedSignature) + assert result.program.signature.equals(ExpectedSignature) + + assert result.scores == [0, 0] + + +def test_opt_composed(): + class MyModule(dspy.Module): + def __init__(self): + self.p1 = TypedPredictor("question:str -> considerations:list[str]", max_retries=1) + self.p2 = TypedPredictor("considerations:list[str] -> answer:str", max_retries=1) + + def forward(self, question): + considerations = self.p1(question=question).considerations + return self.p2(considerations=considerations) + + class ExpectedSignature1(dspy.Signature): + "I1" + + question: str = dspy.InputField(desc="$q", prefix="Q:") + considerations: list[str] = dspy.OutputField(desc="$c", prefix="C:") + + info1 = make_info(ExpectedSignature1) + + class ExpectedSignature2(dspy.Signature): + "I2" + + considerations: list[str] = dspy.InputField(desc="$c", prefix="C:") + answer: str = dspy.OutputField(desc="$a", prefix="A:") + + info2 = make_info(ExpectedSignature2) + + T = TypeVar("T") + + class OutputWrapper(pydantic.BaseModel, Generic[T]): + value: list[T] + + qa_model = DummyLM([]) + prompt_model = DummyLM( + [ + "some thoughts", + OutputWrapper[type(info1)](value=[info1]).model_dump_json(), + "some thoughts", + OutputWrapper[type(info2)](value=[info2]).model_dump_json(), + ] + ) + dspy.settings.configure(lm=qa_model) + + result = optimize_signature( + student=MyModule(), + evaluator=lambda x: 0, # We don't care about the evaluator here + initial_prompts=1, + n_iterations=2, + verbose=True, + prompt_model=prompt_model, + strategy="last", + ) + + assert result.program.p1.signature.equals(ExpectedSignature1) + assert result.program.p2.signature.equals(ExpectedSignature2)