Merge pull request Azure-Samples#63 from qubitron/main

Add more complete implementation of evaluators and tracing

Author: cassiebreviu

.github/workflows/azure-dev.yml

@@ -24,6 +24,7 @@ jobs:
       AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}
       AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}
       AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
+      AZURE_PRINCIPAL_TYPE: 'ServicePrincipal'
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -39,7 +40,7 @@ jobs:
             --federated-credential-provider "github" `
             --tenant-id "$Env:AZURE_TENANT_ID"
         shell: pwsh
-
+        
       - name: Log in with Azure (Client Credentials)
         if: ${{ env.AZURE_CREDENTIALS != '' }}
         run: |

.gitignore

@@ -82,7 +82,7 @@ ipython_config.py
 
 # Environments
 .env
-.venv
+.venv*
 env/
 venv/
 ENV/

.vscode/launch.json

@@ -5,23 +5,20 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "Python: orchestrator.py",
+            "name": "Python: Orchestrator",
             "type": "debugpy",
             "request": "launch",
-            "program": "orchestrator.py",
-            "cwd": "${workspaceFolder}/src/api/api/agents",
-            "console": "integratedTerminal",
+            "module": "api.agents.orchestrator",
+            "cwd": "${workspaceFolder}/src/api",
             "justMyCode": true
         },
         {
-            "name": "Python: evaluate.py",
+            "name": "Python: Evaluate",
             "type": "debugpy",
             "request": "launch",
-            "program": "evaluate.py",
-            "cwd": "${workspaceFolder}/src/api/evaluate",
-            "console": "integratedTerminal",
+            "module": "api.evaluate.evaluate",
+            "cwd": "${workspaceFolder}/src/api",
             "justMyCode": true
-
-        }
+        },
     ]
 }

azure.yaml

@@ -59,8 +59,6 @@ hooks:
 infra:
   provider: terraform
 pipeline:
-  variables:
-    pipeline:
   variables:
     - AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME
     - AZURE_OPENAI_35_TURBO_MODEL_NAME

src/api/.env.sample

@@ -9,3 +9,4 @@ BING_SEARCH_ENDPOINT=https://api.bing.microsoft.com/
 BING_SEARCH_KEY=
 APPINSIGHTS_CONNECTION_STRING=
 #WRITE_EVAL_DATA=true
+#PROMPTFLOW_TRACING_SERVER=true

src/api/api/__init__.py

@@ -1,35 +0,0 @@
-from flask import Flask
-import logging
-import api.get_article as get_article
-
-import os
-from opentelemetry import trace
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter
-from promptflow.tracing import start_trace
-
-def create_app():
-    app = Flask(__name__)
-    app.register_blueprint(get_article.bp)
-    init_logging()
-    return app
-
-
-def init_logging():
-    """Initializes logging."""
-
-    # log to app insights if configured
-    if 'APPINSIGHTS_CONNECTION_STRING' in os.environ:
-        connection_string=os.environ['APPINSIGHTS_CONNECTION_STRING']
-        trace.set_tracer_provider(TracerProvider())
-        trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(AzureMonitorTraceExporter(connection_string=connection_string)))
-
-    #  start pf tracing
-    start_trace()
-
-
-    logging.basicConfig(
-        level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
-    )
-    logging.info("Logging initialized.")

src/api/api/agents/designer/designer.py

@@ -4,7 +4,7 @@
 
 load_dotenv()
 
-def design(context, instructions, feedback):
+def design(request, instructions, feedback):
     # Load prompty with AzureOpenAIModelConfiguration override
     configuration = AzureOpenAIModelConfiguration(
         azure_deployment=os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME"),
@@ -21,7 +21,7 @@ def design(context, instructions, feedback):
         "editor.prompty", model=override_model)
 
     result = prompty_obj(
-        context=context,
+        context=request,
         instructions=instructions,
         feedback=feedback,
     )
@@ -30,7 +30,7 @@ def design(context, instructions, feedback):
 
 if __name__ == "__main__":
     result = design(
-        "The context for the designer.",
+        "The request for the designer.",
         "The instructions for the designer.",
         "The feedback for the designer.")
     print(result)

src/api/api/agents/editor/editor.py

@@ -9,7 +9,7 @@
 load_dotenv()
 
 def edit(article, feedback):
-
+    
     # Load prompty with AzureOpenAIModelConfiguration override
     configuration = AzureOpenAIModelConfiguration(
         azure_deployment=os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME"),
@@ -24,7 +24,6 @@ def edit(article, feedback):
     path_to_prompty = folder + "/editor.prompty"
 
     prompty_obj = Prompty.load(path_to_prompty, model=override_model)
-    
     result = prompty_obj(article=article, feedback=feedback,)
 
     return result

src/api/api/agents/orchestrator.py

@@ -1,18 +1,21 @@
 import json
-from promptflow.tracing import trace, start_trace
+import logging
+from promptflow.tracing import trace
 from api.agents.researcher import researcher
 from api.agents.writer import writer
 from api.agents.editor import editor
 from api.agents.designer import designer
 from api.agents.product import product
+from api.logging import log_output
+from api.evaluate.evaluators import evaluate_article_in_background
+
 from dotenv import load_dotenv
 load_dotenv()
 
 @trace
-def get_research(context, instructions, feedback):
-
+def get_research(request, instructions, feedback):
     research_result = researcher.research(
-        context=context,
+        request=request,
         instructions=instructions,
         feedback=feedback
     )
@@ -23,10 +26,9 @@ def get_research(context, instructions, feedback):
 
 
 @trace
-def get_writer(context, feedback, instructions, research=[], products=[]):
-
+def get_writer(request, feedback, instructions, research=[], products=[]):
     writer_reponse = writer.write(
-        context=context, feedback=feedback, instructions=instructions, research=research, products=products
+        request=request, feedback=feedback, instructions=instructions, research=research, products=products
     )
     print(json.dumps(writer_reponse, indent=2))
     return writer_reponse
@@ -41,12 +43,13 @@ def get_editor(article, feedback):
 
 
 @trace
-def get_designer(context, instructions, feedback):
-    designer_task = designer.design(context, instructions, feedback)
+def get_designer(request, instructions, feedback):
+    designer_task = designer.design(request, instructions, feedback)
     print(json.dumps(designer_task, indent=2))
     return designer_task
 
 
+# TODO: delete, I dont think this is used...
 @trace
 def regenerate_process(editor_response, context, instructions, product_documenation):
     # Get feedback for research from writer
@@ -72,67 +75,78 @@ def regenerate_process(editor_response, context, instructions, product_documenat
     )
     return editor_response
 
-
 @trace
-def get_article(context, instructions):
-    # This code is dup in api response to ueild steped results. TODO: Fix this so its not dup later
+def write_article(request, instructions, evaluate=False):
+    log_output("Article generation started for request: %s, instructions: %s", request, instructions)
+
+    feedback = "No Feedback"
+
+    # Researcher task look up the info
+    yield ("message", "Starting research agent task...")
+    log_output("Getting researcher task output...")
+    research_result = get_research(request, instructions, feedback)
+    yield ("researcher", research_result)
+
+    # Retrieve product information relevant to the user's query
+    log_output("Product information...")
+    product_documenation = product.get_products(request)
+    yield ("products", product_documenation)
+
+    # Then send it to the writer, the writer writes the article
+    yield ("message", "Starting writer agent task...")
+    log_output("Getting writer task output...")
+    writer_response = get_writer(request, feedback, instructions, research=research_result, products=product_documenation)
+    yield ("writer", writer_response)
+
+    # Then send it to the editor, to decide if it's good or not
+    yield ("message", "Starting editor agent task...")
+    log_output("Getting editor task output...")
+    editor_response = get_editor(writer_response["article"], writer_response["feedback"])
+    log_output("Editor response: %s", editor_response)
+
+    yield ("editor", editor_response)
+    retry_count = 0
+    while(str(editor_response["decision"]).lower().startswith("accept")):
+        yield ("message", f"Sending editor feedback ({retry_count + 1})...")
+        log_output("Regeneration attempt %d based on editor feedback", retry_count + 1)
+
+        # Regenerate with feedback loop
+        researchFeedback = editor_response.get("researchFeedback", "No Feedback")
+        editorFeedback = editor_response.get("editorFeedback", "No Feedback")
+        
+        research_result = get_research(request, instructions, researchFeedback)
+        yield ("researcher", research_result)
+
+        writer_response = get_writer(request, editorFeedback, instructions, research=research_result, products=product_documenation)
+        yield ("writer", writer_response)
+
+        editor_response = get_editor(writer_response["article"], writer_response["feedback"])
+        yield ("editor", editor_response)
+
+        retry_count += 1
+        if retry_count >= 2:
+            break
+
+    log_output("Editor accepted article after %d iterations", retry_count)
+    yield ("message", "Editor accepted article")
+
+    if evaluate:
+        evaluate_article_in_background(
+            request=request,
+            instructions=instructions,
+            research=research_result,
+            products=product_documenation,
+            article=writer_response
+        )
+
+    # Log final editor response
+    log_output("Final editor response: %s", json.dumps(editor_response, indent=2))
 
-    feedback = ""
-    print("Getting article for context: ", context)
-
-    # researcher task look up the info
-    print("Getting researcher task output...")
-    research_result = get_research(context, instructions, feedback)
-    product_documenation = product.get_products(context)
-
-    # then send it to the writer, the writer writes the article
-    print("Getting writer task output...")
-    writer_reponse = get_writer(
-        context, feedback, instructions, research=research_result, products=product_documenation
-    )
-
-    # then send it to the editor, to decide if it's good or not
-    print("Getting editor task output...")
-    editor_response = get_editor(
-        writer_reponse["article"], writer_reponse["feedback"]
-    )
-    print(editor_response)
-
-    # retry until decision is accept or until 2x tries
-    if editor_response["decision"] == "reject":
-        print("Editor rejected writer, sending back to writer (1)...")
-        # retry research, writer, and editor with feedback from writer and editor
-        editor_response = regenerate_process(editor_response, context, instructions, product_documenation)
-
-        if editor_response["decision"] == "reject":
-            print("Editor rejected writer again, sending back to writer (2)...")
-            # retry research, writer, and editor with feedback from writer and editor
-            editor_response = regenerate_process(editor_response, context, product_documenation)
-
-    print("Editor accepted writer and research, sending to designer...")
-    # SETH TODO: send to designer
-    # designer_task = designer.design(context, instructions, feedback)
-    # create result object with editor response and writer response
-    result = {"editor_response": editor_response, "writer_response": writer_reponse}
-    print(json.dumps(result, indent=2))
-    return result
- 
 if __name__ == "__main__":
-    import os
-    from opentelemetry import trace
-    from opentelemetry.sdk.trace import TracerProvider
-    from opentelemetry.sdk.trace.export import BatchSpanProcessor
-    from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter
-
-    # log to app insights if configured
-    if 'APPINSIGHTS_CONNECTION_STRING' in os.environ:
-        connection_string=os.environ['APPINSIGHTS_CONNECTION_STRING']
-        trace.set_tracer_provider(TracerProvider())
-        trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(AzureMonitorTraceExporter(connection_string=connection_string)))
-
-    #  start pf tracing
-    start_trace()
+    from api.logging import init_logging
 
+    init_logging()
     context = "Can you find the latest camping trends and what folks are doing in the winter?"
     instructions = "Can you find the relevant information needed and good places to visit"
-    get_article(context, instructions)
+    for result in write_article(context, instructions, evaluate=True):
+        print(*result)

src/api/api/agents/product/ai_search.py

@@ -11,7 +11,7 @@
 
 
 def retrieve_documentation(
-    question: str,
+    request: str,
     index_name: str,
     embedding: List[float],
 ) -> str:
@@ -27,7 +27,7 @@ def retrieve_documentation(
     )
 
     results = search_client.search(
-        search_text=question,
+        search_text=request,
         vector_queries=[vector_query],
         query_type=QueryType.SEMANTIC,
         semantic_configuration_name="default",