refactor app, evaluate can now run and evaluate entire app

Author: qubitron

src/api/api/__init__.py

@@ -1,40 +0,0 @@
-from flask import Flask
-import logging
-import api.get_article as get_article
-
-import os
-from opentelemetry import trace
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from opentelemetry.sdk.trace.sampling import ParentBasedTraceIdRatio
-from promptflow.tracing._integrations._openai_injector import inject_openai_api
-from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter
-from promptflow.tracing import start_trace
-
-def create_app():
-    app = Flask(__name__)
-    app.register_blueprint(get_article.bp)
-    init_logging()
-    return app
-
-
-def init_logging():
-    """Initializes logging."""
-
-    # log to app insights if configured
-    if 'APPLICATIONINSIGHTS_CONNECTION_STRING' in os.environ:
-        inject_openai_api()
-        
-        connection_string=os.environ['APPLICATIONINSIGHTS_CONNECTION_STRING']
-        trace.set_tracer_provider(TracerProvider(sampler=ParentBasedTraceIdRatio(1.0)))
-        trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(AzureMonitorTraceExporter(connection_string=connection_string)))
-
-    if 'PROMPTFLOW_TRACING_SERVER' in os.environ and os.environ['PROMPTFLOW_TRACING_SERVER'] != 'false':
-        start_trace()
-
-    
-
-    logging.basicConfig(
-        level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
-    )
-    logging.info("Logging initialized.")

src/api/api/agents/designer/designer.py

@@ -4,7 +4,7 @@
 
 load_dotenv()
 
-def design(context, instructions, feedback):
+def design(request, instructions, feedback):
     # Load prompty with AzureOpenAIModelConfiguration override
     configuration = AzureOpenAIModelConfiguration(
         azure_deployment=os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME"),
@@ -21,7 +21,7 @@ def design(context, instructions, feedback):
         "editor.prompty", model=override_model)
 
     result = prompty_obj(
-        context=context,
+        context=request,
         instructions=instructions,
         feedback=feedback,
     )
@@ -30,7 +30,7 @@ def design(context, instructions, feedback):
 
 if __name__ == "__main__":
     result = design(
-        "The context for the designer.",
+        "The request for the designer.",
         "The instructions for the designer.",
         "The feedback for the designer.")
     print(result)

src/api/api/agents/editor/editor.py

@@ -9,7 +9,7 @@
 load_dotenv()
 
 def edit(article, feedback):
-
+    
     # Load prompty with AzureOpenAIModelConfiguration override
     configuration = AzureOpenAIModelConfiguration(
         azure_deployment=os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME"),
@@ -24,7 +24,6 @@ def edit(article, feedback):
     path_to_prompty = folder + "/editor.prompty"
 
     prompty_obj = Prompty.load(path_to_prompty, model=override_model)
-    
     result = prompty_obj(article=article, feedback=feedback,)
 
     return result

src/api/api/agents/orchestrator.py

@@ -1,18 +1,21 @@
 import json
-from promptflow.tracing import trace, start_trace
+import logging
+from promptflow.tracing import trace
 from api.agents.researcher import researcher
 from api.agents.writer import writer
 from api.agents.editor import editor
 from api.agents.designer import designer
 from api.agents.product import product
+from api.logging import log_output
+from api.evaluate.evaluators import evaluate_article_in_background
+
 from dotenv import load_dotenv
 load_dotenv()
 
 @trace
-def get_research(context, instructions, feedback):
-
+def get_research(request, instructions, feedback):
     research_result = researcher.research(
-        context=context,
+        request=request,
         instructions=instructions,
         feedback=feedback
     )
@@ -23,10 +26,9 @@ def get_research(context, instructions, feedback):
 
 
 @trace
-def get_writer(context, feedback, instructions, research=[], products=[]):
-
+def get_writer(request, feedback, instructions, research=[], products=[]):
     writer_reponse = writer.write(
-        context=context, feedback=feedback, instructions=instructions, research=research, products=products
+        request=request, feedback=feedback, instructions=instructions, research=research, products=products
     )
     print(json.dumps(writer_reponse, indent=2))
     return writer_reponse
@@ -41,12 +43,13 @@ def get_editor(article, feedback):
 
 
 @trace
-def get_designer(context, instructions, feedback):
-    designer_task = designer.design(context, instructions, feedback)
+def get_designer(request, instructions, feedback):
+    designer_task = designer.design(request, instructions, feedback)
     print(json.dumps(designer_task, indent=2))
     return designer_task
 
 
+# TODO: delete, I dont think this is used...
 @trace
 def regenerate_process(editor_response, context, instructions, product_documenation):
     # Get feedback for research from writer
@@ -72,71 +75,78 @@ def regenerate_process(editor_response, context, instructions, product_documenat
     )
     return editor_response
 
-
 @trace
-def get_article(context, instructions):
-    # This code is dup in api response to ueild steped results. TODO: Fix this so its not dup later
-
-    feedback = ""
-    print("Getting article for context: ", context)
-
-    # researcher task look up the info
-    print("Getting researcher task output...")
-    research_result = get_research(context, instructions, feedback)
-    product_documenation = product.get_products(context)
-
-    # then send it to the writer, the writer writes the article
-    print("Getting writer task output...")
-    writer_reponse = get_writer(
-        context, feedback, instructions, research=research_result, products=product_documenation
-    )
+def write_article(request, instructions, evaluate=False):
+    log_output("Article generation started for request: %s, instructions: %s", request, instructions)
+
+    feedback = "No Feedback"
+
+    # Researcher task look up the info
+    yield ("message", "Starting research agent task...")
+    log_output("Getting researcher task output...")
+    research_result = get_research(request, instructions, feedback)
+    yield ("researcher", research_result)
+
+    # Retrieve product information relevant to the user's query
+    log_output("Product information...")
+    product_documenation = product.get_products(request)
+    yield ("products", product_documenation)
+
+    # Then send it to the writer, the writer writes the article
+    yield ("message", "Starting writer agent task...")
+    log_output("Getting writer task output...")
+    writer_response = get_writer(request, feedback, instructions, research=research_result, products=product_documenation)
+    yield ("writer", writer_response)
+
+    # Then send it to the editor, to decide if it's good or not
+    yield ("message", "Starting editor agent task...")
+    log_output("Getting editor task output...")
+    editor_response = get_editor(writer_response["article"], writer_response["feedback"])
+    log_output("Editor response: %s", editor_response)
+
+    yield ("editor", editor_response)
+    retry_count = 0
+    while(str(editor_response["decision"]).lower().startswith("accept")):
+        yield ("message", f"Sending editor feedback ({retry_count + 1})...")
+        log_output("Regeneration attempt %d based on editor feedback", retry_count + 1)
+
+        # Regenerate with feedback loop
+        researchFeedback = editor_response.get("researchFeedback", "No Feedback")
+        editorFeedback = editor_response.get("editorFeedback", "No Feedback")
+        
+        research_result = get_research(request, instructions, researchFeedback)
+        yield ("researcher", research_result)
+
+        writer_response = get_writer(request, editorFeedback, instructions, research=research_result, products=product_documenation)
+        yield ("writer", writer_response)
+
+        editor_response = get_editor(writer_response["article"], writer_response["feedback"])
+        yield ("editor", editor_response)
+
+        retry_count += 1
+        if retry_count >= 2:
+            break
+
+    log_output("Editor accepted article after %d iterations", retry_count)
+    yield ("message", "Editor accepted article")
+
+    if evaluate:
+        evaluate_article_in_background(
+            request=request,
+            instructions=instructions,
+            research=research_result,
+            products=product_documenation,
+            article=writer_response
+        )
+
+    # Log final editor response
+    log_output("Final editor response: %s", json.dumps(editor_response, indent=2))
 
-    # then send it to the editor, to decide if it's good or not
-    print("Getting editor task output...")
-    editor_response = get_editor(
-        writer_reponse["article"], writer_reponse["feedback"]
-    )
-    print(editor_response)
-
-    # retry until decision is accept or until 2x tries
-    if editor_response["decision"] == "reject":
-        print("Editor rejected writer, sending back to writer (1)...")
-        # retry research, writer, and editor with feedback from writer and editor
-        editor_response = regenerate_process(editor_response, context, instructions, product_documenation)
-
-        if editor_response["decision"] == "reject":
-            print("Editor rejected writer again, sending back to writer (2)...")
-            # retry research, writer, and editor with feedback from writer and editor
-            editor_response = regenerate_process(editor_response, context, product_documenation)
-
-    print("Editor accepted writer and research, sending to designer...")
-    # SETH TODO: send to designer
-    # designer_task = designer.design(context, instructions, feedback)
-    # create result object with editor response and writer response
-    result = {"editor_response": editor_response, "writer_response": writer_reponse}
-    print(json.dumps(result, indent=2))
-    return result
- 
 if __name__ == "__main__":
-    import os
-    from opentelemetry import trace
-    from opentelemetry.sdk.trace import TracerProvider
-    from opentelemetry.sdk.trace.export import BatchSpanProcessor
-    from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter
-    from opentelemetry.sdk.trace.sampling import ParentBasedTraceIdRatio
-    from promptflow.tracing._integrations._openai_injector import inject_openai_api
-
-    # log to app insights if configured
-    if 'APPLICATIONINSIGHTS_CONNECTION_STRING' in os.environ:
-        inject_openai_api()
-
-        connection_string=os.environ['APPLICATIONINSIGHTS_CONNECTION_STRING']
-        trace.set_tracer_provider(TracerProvider(sampler=ParentBasedTraceIdRatio(1.0)))
-        trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(AzureMonitorTraceExporter(connection_string=connection_string)))
-
-    if 'PROMPTFLOW_TRACING_SERVER' in os.environ and os.environ['PROMPTFLOW_TRACING_SERVER'] != 'false':
-        start_trace()
+    from api.logging import init_logging
 
+    init_logging()
     context = "Can you find the latest camping trends and what folks are doing in the winter?"
     instructions = "Can you find the relevant information needed and good places to visit"
-    get_article(context, instructions)
+    for result in write_article(context, instructions, evaluate=True):
+        print(*result)

src/api/api/agents/product/ai_search.py

@@ -11,7 +11,7 @@
 
 
 def retrieve_documentation(
-    question: str,
+    request: str,
     index_name: str,
     embedding: List[float],
 ) -> str:
@@ -27,7 +27,7 @@ def retrieve_documentation(
     )
 
     results = search_client.search(
-        search_text=question,
+        search_text=request,
         vector_queries=[vector_query],
         query_type=QueryType.SEMANTIC,
         semantic_configuration_name="default",

src/api/api/agents/product/product.py

@@ -1,6 +1,6 @@
 import os
 import json
-from typing import Dict, List
+from typing import Dict
 from openai import AzureOpenAI
 
 from azure.identity import DefaultAzureCredential, get_bearer_token_provider
@@ -13,10 +13,10 @@
 load_dotenv()
 
 @trace
-def get_context(question, embedding):
-    return retrieve_documentation(question=question, index_name="contoso-products", embedding=embedding)
+def get_context(request, embedding):
+    return retrieve_documentation(request=request, index_name="contoso-products", embedding=embedding)
 
-def get_embedding(question: str):
+def get_embedding(request: str):
     token_provider = get_bearer_token_provider(
         DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"
     )
@@ -28,18 +28,15 @@ def get_embedding(question: str):
     )
 
     return client.embeddings.create(
-            input=question,
+            input=request,
             model="text-embedding-ada-002"
         ).data[0].embedding
 
-
-def get_products(context: str) -> Dict[str, any]:
-    embedding = get_embedding(context)
-    products = get_context(context, embedding)
-    print(products)
+def get_products(request: str) -> Dict[str, any]:
+    embedding = get_embedding(request)
+    products = get_context(request, embedding)
     return products
 
-
 if __name__ == "__main__":
     context = "what kind of jackets do you have?"
     answer = get_products(context)

src/api/api/agents/researcher/researcher.prompty

@@ -44,7 +44,7 @@ Your queries should be descriptive and match the context and feedback provided.
 Use this context to formulate your queries, market, and tools you will use to description
 your research:
 
-{{context}}
+{{request}}
 
 # Feedback
 Use this feedback to help you refine your queries and responses - if there is any feedback:

src/api/api/agents/researcher/researcher.py

@@ -70,7 +70,7 @@ def find_news(query, market="en-US"):
     return articles
 
 @trace
-def execute(context: str, instructions: str, feedback: str = ""):
+def execute(request: str, instructions: str, feedback: str = ""):
     """Assign a research task to a researcher"""
     functions = {
         "find_information": find_information,
@@ -80,7 +80,7 @@ def execute(context: str, instructions: str, feedback: str = ""):
     fns = prompty.execute(
         "researcher.prompty",
         inputs={
-            "context": context,
+            "request": request,
             "instructions": instructions,
             "feedback": feedback,
         },
@@ -129,8 +129,8 @@ def process(research):
     }
 
 
-def research(context, instructions, feedback: str = ""):
-    r = execute(context=context, instructions=instructions, feedback=feedback)
+def research(request, instructions, feedback: str = ""):
+    r = execute(request=request, instructions=instructions, feedback=feedback)
     p = process(r)
     return p