Issues when using Async + VCR

[izaguirrejoe]

I'm running into an issue when using Async, RubyLLM + VCR. I'm using Async to generate multiple embeddings simultaneously, then attempting to replay using VCR in tests. On the first run (when no cassette is available) the embeddings are generated and saved successfully. On subsequent runs, when using the available cassette, the embeddings are shuffled and stored incorrectly. Seems like the fibers are getting crossed. I saw a (potentially) related issue here, but I'm not using async-http.

I notice that RubyLLM is using Faraday under the hood. There's some strange interplay between Async + VCR + Webmock + Faraday, it seems.

Here's a minimally reproducible script that shows the issue:

require "async"
require "async/barrier"
require "vcr"
require 'test/unit'
require 'webmock/test_unit'
require "ruby_llm"
require 'bundler/inline'

gemfile do
  source 'https://rubygems.org'
  gem 'async'
  gem 'vcr'
  gem 'webmock'
  gem 'ruby_llm'
end

RubyLLM.configure do |config|
  config.openai_api_key = ENV["OPENAI_API_KEY"]
end

VCR.configure do |config|
  config.cassette_library_dir = "fixtures/vcr_cassettes"
  config.filter_sensitive_data('<OPENAI_API_KEY>') { ENV["OPENAI_API_KEY"] }
  config.hook_into :webmock
end

class AsyncVCRTest < Test::Unit::TestCase
  def test_async_faraday
    Sync do
      VCR.use_cassette("async") do
        result = {}
        barrier = Async::Barrier.new

        %w[dog cat turtle elephant flower].each do |term|
          barrier.async do
            embedding = RubyLLM.embed term
            # Let's grab the first term of the vector to compare later.
            result[term] = embedding.vectors.first
          end
        end
        barrier.wait

        pp result
        # This is an example second-run, after the cassette has been saved.
        # Notice that values have been swapped!
        # {"dog" => 0.046830196, "cat" => 0.028151099, "turtle" => 0.016412826, "elephant" => 0.05113775, "flower" => 0.02552942}

        # These are the correct values
        # Obtained from not using VCR, or the first-run when using VCR
        {
          "dog" => 0.051114134,
          "cat" => 0.02552942,
          "turtle" => 0.028160911,
          "elephant" => 0.046814237,
          "flower" => 0.016412826
        }.each do |key, expected|
          actual = result[key]
          # This will pass on the VCR first-run
          # and fail on subsequent runs.
          # There's some variability when creating embeddings, so
          # testing within a certain delta
          assert_in_delta(expected, actual, 0.01)
        end
      end
    end
  end
end

[ioquatix]

Thanks, I will take a look. Sorry, have been pretty busy recently.

[ioquatix]

Here is the code that I used:

require "bundler/inline"

gemfile do
  source 'https://rubygems.org'
  gem 'async'
  gem 'vcr'
  gem 'webmock'
  gem 'ruby_llm'
  gem "test-unit"
end

require "async"
require "async/barrier"
require "vcr"
require 'test/unit'
require 'webmock/test_unit'
require "ruby_llm"
require 'bundler/inline'

RubyLLM.configure do |config|
  config.openai_api_key = ENV["OPENAI_API_KEY"]
end

VCR.configure do |config|
  config.cassette_library_dir = "fixtures/vcr_cassettes"
  config.filter_sensitive_data('<OPENAI_API_KEY>') { ENV["OPENAI_API_KEY"] }
  config.hook_into :webmock
end

class AsyncVCRTest < Test::Unit::TestCase
  def test_async_faraday
    Sync do
      VCR.use_cassette("async") do
        result = {}
        barrier = Async::Barrier.new

        %w[dog cat turtle elephant flower].each do |term|
          barrier.async do
            embedding = RubyLLM.embed term
            # Let's grab the first term of the vector to compare later.
            result[term] = embedding.vectors.first
          end
        end
        barrier.wait

        pp result
        # This is an example second-run, after the cassette has been saved.
        # Notice that values have been swapped!
        # {"dog" => 0.046830196, "cat" => 0.028151099, "turtle" => 0.016412826, "elephant" => 0.05113775, "flower" => 0.02552942}

        # These are the correct values
        # Obtained from not using VCR, or the first-run when using VCR
        {
          "dog" => 0.051114134,
          "cat" => 0.02552942,
          "turtle" => 0.028160911,
          "elephant" => 0.046814237,
          "flower" => 0.016412826
        }.each do |key, expected|
          actual = result[key]
          # This will pass on the VCR first-run
          # and fail on subsequent runs.
          # There's some variability when creating embeddings, so
          # testing within a certain delta
          assert_in_delta(expected, actual, 0.01)
        end
      end
    end
  end
end

With the following VCR cassette:

---
http_interactions:
- request:
    method: post
    uri: https://api.openai.com/v1/embeddings
    body:
      encoding: UTF-8
      string: '{"model":"text-embedding-3-small","input":"dog"}'
    headers:
      Authorization:
      - Bearer <OPENAI_API_KEY>
  response:
    status:
      code: 200
      message: OK
    headers:
      Content-Type:
      - application/json
    body:
      encoding: UTF-8
      string: |
        {
          "object": "list",
          "data": [
            {
              "object": "embedding",
              "index": 0,
              "embedding": [0.051114134, 0.0, 0.0, 0.0]
            }
          ],
          "model": "text-embedding-3-small",
          "usage": {
            "prompt_tokens": 5,
            "total_tokens": 5
          }
        }
  recorded_at: Sun, 27 Apr 2025 08:24:11 GMT

- request:
    method: post
    uri: https://api.openai.com/v1/embeddings
    body:
      encoding: UTF-8
      string: '{"model":"text-embedding-3-small","input":"cat"}'
    headers:
      Authorization:
      - Bearer <OPENAI_API_KEY>
  response:
    status:
      code: 200
      message: OK
    headers:
      Content-Type:
      - application/json
    body:
      encoding: UTF-8
      string: |
        {
          "object": "list",
          "data": [
            {
              "object": "embedding",
              "index": 0,
              "embedding": [0.02552942, 0.0, 0.0, 0.0]
            }
          ],
          "model": "text-embedding-3-small",
          "usage": {
            "prompt_tokens": 5,
            "total_tokens": 5
          }
        }
  recorded_at: Sun, 27 Apr 2025 08:24:11 GMT

- request:
    method: post
    uri: https://api.openai.com/v1/embeddings
    body:
      encoding: UTF-8
      string: '{"model":"text-embedding-3-small","input":"turtle"}'
    headers:
      Authorization:
      - Bearer <OPENAI_API_KEY>
  response:
    status:
      code: 200
      message: OK
    headers:
      Content-Type:
      - application/json
    body:
      encoding: UTF-8
      string: |
        {
          "object": "list",
          "data": [
            {
              "object": "embedding",
              "index": 0,
              "embedding": [0.028160911, 0.0, 0.0, 0.0]
            }
          ],
          "model": "text-embedding-3-small",
          "usage": {
            "prompt_tokens": 5,
            "total_tokens": 5
          }
        }
  recorded_at: Sun, 27 Apr 2025 08:24:11 GMT

- request:
    method: post
    uri: https://api.openai.com/v1/embeddings
    body:
      encoding: UTF-8
      string: '{"model":"text-embedding-3-small","input":"elephant"}'
    headers:
      Authorization:
      - Bearer <OPENAI_API_KEY>
  response:
    status:
      code: 200
      message: OK
    headers:
      Content-Type:
      - application/json
    body:
      encoding: UTF-8
      string: |
        {
          "object": "list",
          "data": [
            {
              "object": "embedding",
              "index": 0,
              "embedding": [0.046814237, 0.0, 0.0, 0.0]
            }
          ],
          "model": "text-embedding-3-small",
          "usage": {
            "prompt_tokens": 5,
            "total_tokens": 5
          }
        }
  recorded_at: Sun, 27 Apr 2025 08:24:11 GMT

- request:
    method: post
    uri: https://api.openai.com/v1/embeddings
    body:
      encoding: UTF-8
      string: '{"model":"text-embedding-3-small","input":"flower"}'
    headers:
      Authorization:
      - Bearer <OPENAI_API_KEY>
  response:
    status:
      code: 200
      message: OK
    headers:
      Content-Type:
      - application/json
    body:
      encoding: UTF-8
      string: |
        {
          "object": "list",
          "data": [
            {
              "object": "embedding",
              "index": 0,
              "embedding": [0.016412826, 0.0, 0.0, 0.0]
            }
          ],
          "model": "text-embedding-3-small",
          "usage": {
            "prompt_tokens": 5,
            "total_tokens": 5
          }
        }
  recorded_at: Sun, 27 Apr 2025 08:24:11 GMT

recorded_with: VCR 6.3.1

No matter how many times I run it, I get the same output:

samuel@MacBookPro ~/D/s/a/e/vcr (main)> ruby ./test.rb
Loaded suite ./test
Started
{"dog" => 0.051114134, "cat" => 0.02552942, "turtle" => 0.028160911, "elephant" => 0.046814237, "flower" => 0.016412826}
Finished in 0.018346 seconds.
-------------------------------------------------------------------------------------------------------------------------------------------
1 tests, 5 assertions, 0 failures, 0 errors, 0 pendings, 0 omissions, 0 notifications
100% passed
-------------------------------------------------------------------------------------------------------------------------------------------
54.51 tests/s, 272.54 assertions/s
samuel@MacBookPro ~/D/s/a/e/vcr (main)> ruby ./test.rb
Loaded suite ./test
Started
{"dog" => 0.051114134, "cat" => 0.02552942, "turtle" => 0.028160911, "elephant" => 0.046814237, "flower" => 0.016412826}
Finished in 0.01819 seconds.
-------------------------------------------------------------------------------------------------------------------------------------------
1 tests, 5 assertions, 0 failures, 0 errors, 0 pendings, 0 omissions, 0 notifications
100% passed
-------------------------------------------------------------------------------------------------------------------------------------------
54.98 tests/s, 274.88 assertions/s

Is this consistent with your experience?

[izaguirrejoe]

Interesting: using your cassette fixture, my test does indeed pass. Comparing with my cassette, it contains the same HTTP interactions but in a different order. Yours is in the order of my array:

dog cat turtle elephant flower

mine is recorded in this order (but the corresponding embeddings are the same as yours, only the order of HTTP request in cassette is different):

elephant, turtle, flower, dog, cat.

If I take your cassette and swap the order of the first and second interactions, I get the same failing tests that I've been getting. I've read the VCR documentation and it doesn't seem to mention the order at all. Looking at VCR source code, it seems like order shouldn't matter, but I'm not 100% sure. I'd expect it shouldn't matter if VCR is to properly handle asynchronous code like Async, no?

This is your cassette, but with the first and second HTTP interactions swapped. This is failing on my machine:

---
http_interactions:
- request:
    method: post
    uri: https://api.openai.com/v1/embeddings
    body:
      encoding: UTF-8
      string: '{"model":"text-embedding-3-small","input":"cat"}'
    headers:
      Authorization:
      - Bearer <OPENAI_API_KEY>
  response:
    status:
      code: 200
      message: OK
    headers:
      Content-Type:
      - application/json
    body:
      encoding: UTF-8
      string: |
        {
          "object": "list",
          "data": [
            {
              "object": "embedding",
              "index": 0,
              "embedding": [0.02552942, 0.0, 0.0, 0.0]
            }
          ],
          "model": "text-embedding-3-small",
          "usage": {
            "prompt_tokens": 5,
            "total_tokens": 5
          }
        }
  recorded_at: Sun, 27 Apr 2025 08:24:11 GMT

- request:
    method: post
    uri: https://api.openai.com/v1/embeddings
    body:
      encoding: UTF-8
      string: '{"model":"text-embedding-3-small","input":"dog"}'
    headers:
      Authorization:
      - Bearer <OPENAI_API_KEY>
  response:
    status:
      code: 200
      message: OK
    headers:
      Content-Type:
      - application/json
    body:
      encoding: UTF-8
      string: |
        {
          "object": "list",
          "data": [
            {
              "object": "embedding",
              "index": 0,
              "embedding": [0.051114134, 0.0, 0.0, 0.0]
            }
          ],
          "model": "text-embedding-3-small",
          "usage": {
            "prompt_tokens": 5,
            "total_tokens": 5
          }
        }
  recorded_at: Sun, 27 Apr 2025 08:24:11 GMT

- request:
    method: post
    uri: https://api.openai.com/v1/embeddings
    body:
      encoding: UTF-8
      string: '{"model":"text-embedding-3-small","input":"turtle"}'
    headers:
      Authorization:
      - Bearer <OPENAI_API_KEY>
  response:
    status:
      code: 200
      message: OK
    headers:
      Content-Type:
      - application/json
    body:
      encoding: UTF-8
      string: |
        {
          "object": "list",
          "data": [
            {
              "object": "embedding",
              "index": 0,
              "embedding": [0.028160911, 0.0, 0.0, 0.0]
            }
          ],
          "model": "text-embedding-3-small",
          "usage": {
            "prompt_tokens": 5,
            "total_tokens": 5
          }
        }
  recorded_at: Sun, 27 Apr 2025 08:24:11 GMT

- request:
    method: post
    uri: https://api.openai.com/v1/embeddings
    body:
      encoding: UTF-8
      string: '{"model":"text-embedding-3-small","input":"elephant"}'
    headers:
      Authorization:
      - Bearer <OPENAI_API_KEY>
  response:
    status:
      code: 200
      message: OK
    headers:
      Content-Type:
      - application/json
    body:
      encoding: UTF-8
      string: |
        {
          "object": "list",
          "data": [
            {
              "object": "embedding",
              "index": 0,
              "embedding": [0.046814237, 0.0, 0.0, 0.0]
            }
          ],
          "model": "text-embedding-3-small",
          "usage": {
            "prompt_tokens": 5,
            "total_tokens": 5
          }
        }
  recorded_at: Sun, 27 Apr 2025 08:24:11 GMT

- request:
    method: post
    uri: https://api.openai.com/v1/embeddings
    body:
      encoding: UTF-8
      string: '{"model":"text-embedding-3-small","input":"flower"}'
    headers:
      Authorization:
      - Bearer <OPENAI_API_KEY>
  response:
    status:
      code: 200
      message: OK
    headers:
      Content-Type:
      - application/json
    body:
      encoding: UTF-8
      string: |
        {
          "object": "list",
          "data": [
            {
              "object": "embedding",
              "index": 0,
              "embedding": [0.016412826, 0.0, 0.0, 0.0]
            }
          ],
          "model": "text-embedding-3-small",
          "usage": {
            "prompt_tokens": 5,
            "total_tokens": 5
          }
        }
  recorded_at: Sun, 27 Apr 2025 08:24:11 GMT

recorded_with: VCR 6.3.1