From 607394b3d2942211fe30737cda94fe437a7814be Mon Sep 17 00:00:00 2001 From: priya-dwivedi Date: Wed, 28 Mar 2018 10:44:57 -0400 Subject: [PATCH] final SQUaD code --- attention_visualization.ipynb | 7649 ----------------------------- code/official_eval_helper_best.py | 281 -- code/qa_model_best.py | 860 ---- codelab_details | 41 - codelab_instructions | 8 - create_small_sets.ipynb | 219 - run_instructions | 127 - 7 files changed, 9185 deletions(-) delete mode 100644 attention_visualization.ipynb delete mode 100644 code/official_eval_helper_best.py delete mode 100644 code/qa_model_best.py delete mode 100644 codelab_details delete mode 100644 codelab_instructions delete mode 100644 create_small_sets.ipynb delete mode 100644 run_instructions diff --git a/attention_visualization.ipynb b/attention_visualization.ipynb deleted file mode 100644 index eaa4e27..0000000 --- a/attention_visualization.ipynb +++ /dev/null @@ -1,7649 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 97, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import json\n", - "from pprint import pprint\n", - "\n", - "from __future__ import print_function\n", - "from collections import Counter\n", - "import string\n", - "import re\n", - "import argparse\n", - "import json\n", - "import sys\n" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{u'data': [{u'paragraphs': [{u'context': u'Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24\\u201310 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi\\'s Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the \"golden anniversary\" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as \"Super Bowl L\"), so that the logo could prominently feature the Arabic numerals 50.',\n", - " u'qas': [{u'answers': [{u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'}],\n", - " u'id': u'56be4db0acb8001400a502ec',\n", - " u'question': u'Which NFL team represented the AFC at Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 249,\n", - " u'text': u'Carolina Panthers'},\n", - " {u'answer_start': 249,\n", - " u'text': u'Carolina Panthers'},\n", - " {u'answer_start': 249,\n", - " u'text': u'Carolina Panthers'}],\n", - " u'id': u'56be4db0acb8001400a502ed',\n", - " u'question': u'Which NFL team represented the NFC at Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 403,\n", - " u'text': u'Santa Clara, California'},\n", - " {u'answer_start': 355,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 355,\n", - " u'text': u\"Levi's Stadium in the San Francisco Bay Area at Santa Clara, California.\"}],\n", - " u'id': u'56be4db0acb8001400a502ee',\n", - " u'question': u'Where did Super Bowl 50 take place?'},\n", - " {u'answers': [{u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'}],\n", - " u'id': u'56be4db0acb8001400a502ef',\n", - " u'question': u'Which NFL team won Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 488,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 488,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 521,\n", - " u'text': u'gold'}],\n", - " u'id': u'56be4db0acb8001400a502f0',\n", - " u'question': u'What color was used to emphasize the 50th anniversary of the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 487,\n", - " u'text': u'\"golden anniversary\"'},\n", - " {u'answer_start': 521,\n", - " u'text': u'gold-themed'},\n", - " {u'answer_start': 487,\n", - " u'text': u'\"golden anniversary'}],\n", - " u'id': u'56be8e613aeaaa14008c90d1',\n", - " u'question': u'What was the theme of Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 334,\n", - " u'text': u'February 7, 2016'},\n", - " {u'answer_start': 334,\n", - " u'text': u'February 7'},\n", - " {u'answer_start': 334,\n", - " u'text': u'February 7, 2016'}],\n", - " u'id': u'56be8e613aeaaa14008c90d2',\n", - " u'question': u'What day was the game played on?'},\n", - " {u'answers': [{u'answer_start': 133,\n", - " u'text': u'American Football Conference'},\n", - " {u'answer_start': 133,\n", - " u'text': u'American Football Conference'},\n", - " {u'answer_start': 133,\n", - " u'text': u'American Football Conference'}],\n", - " u'id': u'56be8e613aeaaa14008c90d3',\n", - " u'question': u'What is the AFC short for?'},\n", - " {u'answers': [{u'answer_start': 487,\n", - " u'text': u'\"golden anniversary\"'},\n", - " {u'answer_start': 521,\n", - " u'text': u'gold-themed'},\n", - " {u'answer_start': 521,\n", - " u'text': u'gold'}],\n", - " u'id': u'56bea9923aeaaa14008c91b9',\n", - " u'question': u'What was the theme of Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 133,\n", - " u'text': u'American Football Conference'},\n", - " {u'answer_start': 133,\n", - " u'text': u'American Football Conference'},\n", - " {u'answer_start': 133,\n", - " u'text': u'American Football Conference'}],\n", - " u'id': u'56bea9923aeaaa14008c91ba',\n", - " u'question': u'What does AFC stand for?'},\n", - " {u'answers': [{u'answer_start': 334,\n", - " u'text': u'February 7, 2016'},\n", - " {u'answer_start': 334,\n", - " u'text': u'February 7'},\n", - " {u'answer_start': 334,\n", - " u'text': u'February 7, 2016'}],\n", - " u'id': u'56bea9923aeaaa14008c91bb',\n", - " u'question': u'What day was the Super Bowl played on?'},\n", - " {u'answers': [{u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'}],\n", - " u'id': u'56beace93aeaaa14008c91df',\n", - " u'question': u'Who won Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 355,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 355,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 355,\n", - " u'text': u\"Levi's Stadium in the San Francisco Bay Area at Santa Clara\"}],\n", - " u'id': u'56beace93aeaaa14008c91e0',\n", - " u'question': u'What venue did Super Bowl 50 take place in?'},\n", - " {u'answers': [{u'answer_start': 403,\n", - " u'text': u'Santa Clara'},\n", - " {u'answer_start': 403,\n", - " u'text': u'Santa Clara'},\n", - " {u'answer_start': 403,\n", - " u'text': u'Santa Clara'}],\n", - " u'id': u'56beace93aeaaa14008c91e1',\n", - " u'question': u'What city did Super Bowl 50 take place in?'},\n", - " {u'answers': [{u'answer_start': 693,\n", - " u'text': u'Super Bowl L'},\n", - " {u'answer_start': 704,\n", - " u'text': u'L'},\n", - " {u'answer_start': 693,\n", - " u'text': u'Super Bowl L'}],\n", - " u'id': u'56beace93aeaaa14008c91e2',\n", - " u'question': u'If Roman numerals were used, what would Super Bowl 50 have been called?'},\n", - " {u'answers': [{u'answer_start': 116,\n", - " u'text': u'2015'},\n", - " {u'answer_start': 112,\n", - " u'text': u'the 2015 season'},\n", - " {u'answer_start': 116,\n", - " u'text': u'2015'}],\n", - " u'id': u'56beace93aeaaa14008c91e3',\n", - " u'question': u'Super Bowl 50 decided the NFL champion for what season?'},\n", - " {u'answers': [{u'answer_start': 116,\n", - " u'text': u'2015'},\n", - " {u'answer_start': 346,\n", - " u'text': u'2016'},\n", - " {u'answer_start': 116,\n", - " u'text': u'2015'}],\n", - " u'id': u'56bf10f43aeaaa14008c94fd',\n", - " u'question': u'What year did the Denver Broncos secure a Super Bowl title for the third time?'},\n", - " {u'answers': [{u'answer_start': 403,\n", - " u'text': u'Santa Clara'},\n", - " {u'answer_start': 403,\n", - " u'text': u'Santa Clara'},\n", - " {u'answer_start': 403,\n", - " u'text': u'Santa Clara'}],\n", - " u'id': u'56bf10f43aeaaa14008c94fe',\n", - " u'question': u'What city did Super Bowl 50 take place in?'},\n", - " {u'answers': [{u'answer_start': 355,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 355,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 355,\n", - " u'text': u\"Levi's Stadium\"}],\n", - " u'id': u'56bf10f43aeaaa14008c94ff',\n", - " u'question': u'What stadium did Super Bowl 50 take place in?'},\n", - " {u'answers': [{u'answer_start': 267,\n", - " u'text': u'24\\u201310'},\n", - " {u'answer_start': 267,\n", - " u'text': u'24\\u201310'},\n", - " {u'answer_start': 267,\n", - " u'text': u'24\\u201310'}],\n", - " u'id': u'56bf10f43aeaaa14008c9500',\n", - " u'question': u'What was the final score of Super Bowl 50? '},\n", - " {u'answers': [{u'answer_start': 334,\n", - " u'text': u'February 7, 2016'},\n", - " {u'answer_start': 334,\n", - " u'text': u'February 7, 2016'},\n", - " {u'answer_start': 334,\n", - " u'text': u'February 7, 2016'}],\n", - " u'id': u'56bf10f43aeaaa14008c9501',\n", - " u'question': u'What month, day and year did Super Bowl 50 take place? '},\n", - " {u'answers': [{u'answer_start': 116,\n", - " u'text': u'2015'},\n", - " {u'answer_start': 346,\n", - " u'text': u'2016'},\n", - " {u'answer_start': 346,\n", - " u'text': u'2016'}],\n", - " u'id': u'56d20362e7d4791d009025e8',\n", - " u'question': u'What year was Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'}],\n", - " u'id': u'56d20362e7d4791d009025e9',\n", - " u'question': u'What team was the AFC champion?'},\n", - " {u'answers': [{u'answer_start': 249,\n", - " u'text': u'Carolina Panthers'},\n", - " {u'answer_start': 249,\n", - " u'text': u'Carolina Panthers'},\n", - " {u'answer_start': 249,\n", - " u'text': u'Carolina Panthers'}],\n", - " u'id': u'56d20362e7d4791d009025ea',\n", - " u'question': u'What team was the NFC champion?'},\n", - " {u'answers': [{u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'}],\n", - " u'id': u'56d20362e7d4791d009025eb',\n", - " u'question': u'Who won Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 116,\n", - " u'text': u'2015'},\n", - " {u'answer_start': 112,\n", - " u'text': u'the 2015 season'},\n", - " {u'answer_start': 116,\n", - " u'text': u'2015'}],\n", - " u'id': u'56d600e31c85041400946eae',\n", - " u'question': u'Super Bowl 50 determined the NFL champion for what season?'},\n", - " {u'answers': [{u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'}],\n", - " u'id': u'56d600e31c85041400946eb0',\n", - " u'question': u'Which team won Super Bowl 50.'},\n", - " {u'answers': [{u'answer_start': 403,\n", - " u'text': u'Santa Clara, California.'},\n", - " {u'answer_start': 355,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 355,\n", - " u'text': u\"Levi's Stadium\"}],\n", - " u'id': u'56d600e31c85041400946eb1',\n", - " u'question': u'Where was Super Bowl 50 held?'},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'Super Bowl'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Super Bowl'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Super Bowl'}],\n", - " u'id': u'56d9895ddc89441400fdb50e',\n", - " u'question': u'The name of the NFL championship game is?'},\n", - " {u'answers': [{u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'},\n", - " {u'answer_start': 177,\n", - " u'text': u'Denver Broncos'}],\n", - " u'id': u'56d9895ddc89441400fdb510',\n", - " u'question': u'What 2015 NFL team one the AFC playoff?'}]},\n", - " {u'context': u'The Panthers finished the regular season with a 15\\u20131 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49\\u201315 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12\\u20134 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20\\u201318 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.',\n", - " u'qas': [{u'answers': [{u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'}],\n", - " u'id': u'56be4e1facb8001400a502f6',\n", - " u'question': u'Which Carolina Panthers player was named Most Valuable Player?'},\n", - " {u'answers': [{u'answer_start': 467,\n", - " u'text': u'8'},\n", - " {u'answer_start': 601,\n", - " u'text': u'eight'},\n", - " {u'answer_start': 601,\n", - " u'text': u'eight'}],\n", - " u'id': u'56be4e1facb8001400a502f9',\n", - " u'question': u'How many appearances have the Denver Broncos made in the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 291,\n", - " u'text': u'1995'},\n", - " {u'answer_start': 291,\n", - " u'text': u'1995'},\n", - " {u'answer_start': 291,\n", - " u'text': u'1995'}],\n", - " u'id': u'56be4e1facb8001400a502fa',\n", - " u'question': u'What year was the Carolina Panthers franchise founded?'},\n", - " {u'answers': [{u'answer_start': 152,\n", - " u'text': u'Arizona Cardinals'},\n", - " {u'answer_start': 148,\n", - " u'text': u'the Arizona Cardinals'},\n", - " {u'answer_start': 152,\n", - " u'text': u'Arizona Cardinals'}],\n", - " u'id': u'56beaa4a3aeaaa14008c91c2',\n", - " u'question': u'What team did the Panthers defeat?'},\n", - " {u'answers': [{u'answer_start': 372,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 368,\n", - " u'text': u'the New England Patriots'},\n", - " {u'answer_start': 372,\n", - " u'text': u'New England Patriots'}],\n", - " u'id': u'56beaa4a3aeaaa14008c91c3',\n", - " u'question': u'Who did the Broncos prevent from going to the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 152,\n", - " u'text': u'Arizona Cardinals'},\n", - " {u'answer_start': 148,\n", - " u'text': u'the Arizona Cardinals'},\n", - " {u'answer_start': 152,\n", - " u'text': u'Arizona Cardinals'}],\n", - " u'id': u'56bead5a3aeaaa14008c91e9',\n", - " u'question': u'Who did the Panthers beat in the NFC Championship Game?'},\n", - " {u'answers': [{u'answer_start': 372,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 368,\n", - " u'text': u'the New England Patriots'},\n", - " {u'answer_start': 372,\n", - " u'text': u'New England Patriots'}],\n", - " u'id': u'56bead5a3aeaaa14008c91ea',\n", - " u'question': u'Who lost to the Broncos in the AFC Championship?'},\n", - " {u'answers': [{u'answer_start': 372,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 368,\n", - " u'text': u'the New England Patriots'},\n", - " {u'answer_start': 372,\n", - " u'text': u'New England Patriots'}],\n", - " u'id': u'56bead5a3aeaaa14008c91eb',\n", - " u'question': u'Who were the defending Super Bowl champions?'},\n", - " {u'answers': [{u'answer_start': 575,\n", - " u'text': u'four'},\n", - " {u'answer_start': 575,\n", - " u'text': u'four'},\n", - " {u'answer_start': 575,\n", - " u'text': u'four'}],\n", - " u'id': u'56bead5a3aeaaa14008c91ec',\n", - " u'question': u'How many teams have been in the Super Bowl eight times?'},\n", - " {u'answers': [{u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'}],\n", - " u'id': u'56bead5a3aeaaa14008c91ed',\n", - " u'question': u\"Who was this season's NFL MVP?\"},\n", - " {u'answers': [{u'answer_start': 48,\n", - " u'text': u'15\\u20131'},\n", - " {u'answer_start': 48,\n", - " u'text': u'15\\u20131'},\n", - " {u'answer_start': 48,\n", - " u'text': u'15\\u20131'}],\n", - " u'id': u'56bf159b3aeaaa14008c9507',\n", - " u'question': u'What was the win/loss ratio in 2015 for the Carolina Panthers during their regular season?'},\n", - " {u'answers': [{u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'}],\n", - " u'id': u'56bf159b3aeaaa14008c9508',\n", - " u'question': u\"Which Carolina Panthers team member was picked as the team's MVP in 2015? \"},\n", - " {u'answers': [{u'answer_start': 344,\n", - " u'text': u'12\\u20134'},\n", - " {u'answer_start': 344,\n", - " u'text': u'12\\u20134'},\n", - " {u'answer_start': 344,\n", - " u'text': u'12\\u20134'}],\n", - " u'id': u'56bf159b3aeaaa14008c9509',\n", - " u'question': u\"What were the win/loss game stats for the Denver Bronco's regular season in 2015?\"},\n", - " {u'answers': [{u'answer_start': 170,\n", - " u'text': u'4'},\n", - " {u'answer_start': 575,\n", - " u'text': u'four'},\n", - " {u'answer_start': 575,\n", - " u'text': u'four'}],\n", - " u'id': u'56bf159b3aeaaa14008c950a',\n", - " u'question': u'How many teams have played in the Super Bowl eight times?'},\n", - " {u'answers': [{u'answer_start': 372,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 368,\n", - " u'text': u'the New England Patriots'},\n", - " {u'answer_start': 372,\n", - " u'text': u'New England Patriots'}],\n", - " u'id': u'56bf159b3aeaaa14008c950b',\n", - " u'question': u'Which team did not get a chance to defend their Super Bowl XLIX win in Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'}],\n", - " u'id': u'56d2045de7d4791d009025f3',\n", - " u'question': u'Who is the quarterback for the Panthers?'},\n", - " {u'answers': [{u'answer_start': 152,\n", - " u'text': u'Arizona Cardinals'},\n", - " {u'answer_start': 148,\n", - " u'text': u'the Arizona Cardinals'},\n", - " {u'answer_start': 152,\n", - " u'text': u'Arizona Cardinals'}],\n", - " u'id': u'56d2045de7d4791d009025f4',\n", - " u'question': u'Who did Carolina beat in the NFC championship game?'},\n", - " {u'answers': [{u'answer_start': 345,\n", - " u'text': u'2'},\n", - " {u'answer_start': 227,\n", - " u'text': u'second'},\n", - " {u'answer_start': 227,\n", - " u'text': u'second'}],\n", - " u'id': u'56d2045de7d4791d009025f5',\n", - " u'question': u'How many times have the Panthers been in the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 372,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 368,\n", - " u'text': u'the New England Patriots'},\n", - " {u'answer_start': 372,\n", - " u'text': u'New England Patriots'}],\n", - " u'id': u'56d2045de7d4791d009025f6',\n", - " u'question': u'Who did Denver beat in the AFC championship?'},\n", - " {u'answers': [{u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'}],\n", - " u'id': u'56d6017d1c85041400946ebe',\n", - " u'question': u'Who was the Most Valuable Player for the 2015 NFL season?'},\n", - " {u'answers': [{u'answer_start': 372,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 368,\n", - " u'text': u'the New England Patriots'},\n", - " {u'answer_start': 372,\n", - " u'text': u'New England Patriots'}],\n", - " u'id': u'56d6017d1c85041400946ec1',\n", - " u'question': u'Who did Denver beat in the 2015 AFC Championship game?'},\n", - " {u'answers': [{u'answer_start': 152,\n", - " u'text': u'Arizona Cardinals'},\n", - " {u'answer_start': 148,\n", - " u'text': u'the Arizona Cardinals'},\n", - " {u'answer_start': 152,\n", - " u'text': u'Arizona Cardinals'}],\n", - " u'id': u'56d6017d1c85041400946ec2',\n", - " u'question': u'Who did the Carolina Panthers beat in the 2015 NFC Championship game?'},\n", - " {u'answers': [{u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 77,\n", - " u'text': u'Cam Newton'}],\n", - " u'id': u'56d98a59dc89441400fdb52a',\n", - " u'question': u'Who was the 2015 NFL MVP?'},\n", - " {u'answers': [{u'answer_start': 152,\n", - " u'text': u'Arizona Cardinals'},\n", - " {u'answer_start': 148,\n", - " u'text': u'the Arizona Cardinals'},\n", - " {u'answer_start': 152,\n", - " u'text': u'Arizona Cardinals'}],\n", - " u'id': u'56d98a59dc89441400fdb52b',\n", - " u'question': u'Who did the Panthers beat to become the NFC champs?'},\n", - " {u'answers': [{u'answer_start': 291,\n", - " u'text': u'1995.'},\n", - " {u'answer_start': 291,\n", - " u'text': u'1995'},\n", - " {u'answer_start': 291,\n", - " u'text': u'1995'}],\n", - " u'id': u'56d98a59dc89441400fdb52e',\n", - " u'question': u'What year did the Carolina Panthers form?'}]},\n", - " {u'context': u\"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2\\xbd sacks, and two forced fumbles.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 252,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56be4eafacb8001400a50302',\n", - " u'question': u'Who was the Super Bowl 50 MVP?'},\n", - " {u'answers': [{u'answer_start': 314,\n", - " u'text': u'2'},\n", - " {u'answer_start': 328,\n", - " u'text': u'two'},\n", - " {u'answer_start': 328,\n", - " u'text': u'two'}],\n", - " u'id': u'56be4eafacb8001400a50303',\n", - " u'question': u'How many fumbles did Von Miller force in Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 4,\n", - " u'text': u'Broncos'},\n", - " {u'answer_start': 0,\n", - " u'text': u'The Broncos'},\n", - " {u'answer_start': 4,\n", - " u'text': u'Broncos'}],\n", - " u'id': u'56be4eafacb8001400a50304',\n", - " u'question': u'Which team held the scoring lead throughout the entire game?'},\n", - " {u'answers': [{u'answer_start': 237,\n", - " u'text': u'linebacker Von Miller'},\n", - " {u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 252,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56beab833aeaaa14008c91d2',\n", - " u'question': u'Which Denver linebacker was named Super Bowl MVP?'},\n", - " {u'answers': [{u'answer_start': 295,\n", - " u'text': u'five solo tackles'},\n", - " {u'answer_start': 295,\n", - " u'text': u'five'},\n", - " {u'answer_start': 295,\n", - " u'text': u'five'}],\n", - " u'id': u'56beab833aeaaa14008c91d3',\n", - " u'question': u'How many solo tackles did Von Miller make at Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 67,\n", - " u'text': u\"Newton was limited by Denver's defense\"},\n", - " {u'answer_start': 67,\n", - " u'text': u'Newton'},\n", - " {u'answer_start': 67,\n", - " u'text': u'Newton'}],\n", - " u'id': u'56beab833aeaaa14008c91d4',\n", - " u'question': u\"Who was limited by Denver's defense?\"},\n", - " {u'answers': [{u'answer_start': 124,\n", - " u'text': u'seven'},\n", - " {u'answer_start': 124,\n", - " u'text': u'seven'},\n", - " {u'answer_start': 124,\n", - " u'text': u'seven'}],\n", - " u'id': u'56beae423aeaaa14008c91f4',\n", - " u'question': u'How many times was Cam Newton sacked?'},\n", - " {u'answers': [{u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 0,\n", - " u'text': u'The Broncos'},\n", - " {u'answer_start': 252,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56beae423aeaaa14008c91f5',\n", - " u'question': u'Who won the Super Bowl MVP?'},\n", - " {u'answers': [{u'answer_start': 156,\n", - " u'text': u'three'},\n", - " {u'answer_start': 156,\n", - " u'text': u'three'},\n", - " {u'answer_start': 156,\n", - " u'text': u'three'}],\n", - " u'id': u'56beae423aeaaa14008c91f6',\n", - " u'question': u'How many turnovers did Cam Newton have?'},\n", - " {u'answers': [{u'answer_start': 328,\n", - " u'text': u'two'},\n", - " {u'answer_start': 328,\n", - " u'text': u'two'},\n", - " {u'answer_start': 328,\n", - " u'text': u'two'}],\n", - " u'id': u'56beae423aeaaa14008c91f7',\n", - " u'question': u'How many fumbles did Von Miller force?'},\n", - " {u'answers': [{u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 252,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56bf17653aeaaa14008c9511',\n", - " u'question': u'Who was given the esteemed status of MVP for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 237,\n", - " u'text': u'linebacker'},\n", - " {u'answer_start': 237,\n", - " u'text': u'linebacker'},\n", - " {u'answer_start': 237,\n", - " u'text': u'linebacker'}],\n", - " u'id': u'56bf17653aeaaa14008c9513',\n", - " u'question': u'What position does Von Miller play for the Denver Broncos?'},\n", - " {u'answers': [{u'answer_start': 45,\n", - " u'text': u'5'},\n", - " {u'answer_start': 295,\n", - " u'text': u'five'},\n", - " {u'answer_start': 295,\n", - " u'text': u'five'}],\n", - " u'id': u'56bf17653aeaaa14008c9514',\n", - " u'question': u'What was the number of solo tackles that Von Miller had in Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 314,\n", - " u'text': u'2'},\n", - " {u'answer_start': 328,\n", - " u'text': u'two'},\n", - " {u'answer_start': 328,\n", - " u'text': u'two'}],\n", - " u'id': u'56bf17653aeaaa14008c9515',\n", - " u'question': u'How many forced fumbles did Von Miller have during the Super Bowl 50 game?'},\n", - " {u'answers': [{u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 248,\n", - " u'text': u'Von Miller'}],\n", - " u'id': u'56d204ade7d4791d00902603',\n", - " u'question': u'Who won the MVP for the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 45,\n", - " u'text': u'5'},\n", - " {u'answer_start': 295,\n", - " u'text': u'five'},\n", - " {u'answer_start': 295,\n", - " u'text': u'five'}],\n", - " u'id': u'56d204ade7d4791d00902604',\n", - " u'question': u'How many tackles did Von Miller get during the game?'},\n", - " {u'answers': [{u'answer_start': 124,\n", - " u'text': u'seven'},\n", - " {u'answer_start': 124,\n", - " u'text': u'seven'},\n", - " {u'answer_start': 124,\n", - " u'text': u'seven'}],\n", - " u'id': u'56d601e41c85041400946ece',\n", - " u'question': u'How many times was Cam Newton sacked in Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 156,\n", - " u'text': u'three'},\n", - " {u'answer_start': 156,\n", - " u'text': u'three'},\n", - " {u'answer_start': 156,\n", - " u'text': u'three'}],\n", - " u'id': u'56d601e41c85041400946ecf',\n", - " u'question': u'How many times did the Denver defense force Newton into turnovers?'},\n", - " {u'answers': [{u'answer_start': 183,\n", - " u'text': u'a fumble'},\n", - " {u'answer_start': 183,\n", - " u'text': u'a fumble'},\n", - " {u'answer_start': 185,\n", - " u'text': u'fumble'}],\n", - " u'id': u'56d601e41c85041400946ed0',\n", - " u'question': u'Which Newton turnover resulted in seven points for Denver?'},\n", - " {u'answers': [{u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 248,\n", - " u'text': u'Von Miller'}],\n", - " u'id': u'56d601e41c85041400946ed1',\n", - " u'question': u'Who was the Most Valuable Player of Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 237,\n", - " u'text': u'linebacker'},\n", - " {u'answer_start': 237,\n", - " u'text': u'linebacker'},\n", - " {u'answer_start': 237,\n", - " u'text': u'linebacker'}],\n", - " u'id': u'56d601e41c85041400946ed2',\n", - " u'question': u'What position does Von Miller play?'},\n", - " {u'answers': [{u'answer_start': 124,\n", - " u'text': u'seven'},\n", - " {u'answer_start': 124,\n", - " u'text': u'seven'},\n", - " {u'answer_start': 124,\n", - " u'text': u'seven'}],\n", - " u'id': u'56d98b33dc89441400fdb53b',\n", - " u'question': u\"How many times was the Panthers' quarterback sacked?\"},\n", - " {u'answers': [{u'answer_start': 156,\n", - " u'text': u'three'},\n", - " {u'answer_start': 156,\n", - " u'text': u'three'},\n", - " {u'answer_start': 156,\n", - " u'text': u'three'}],\n", - " u'id': u'56d98b33dc89441400fdb53c',\n", - " u'question': u'How many times did the Broncos cause turnovers in the game?'},\n", - " {u'answers': [{u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 248,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 248,\n", - " u'text': u'Von Miller'}],\n", - " u'id': u'56d98b33dc89441400fdb53d',\n", - " u'question': u'What Denver player caused two fumbles for the Panthers?'},\n", - " {u'answers': [{u'answer_start': 295,\n", - " u'text': u'five'},\n", - " {u'answer_start': 295,\n", - " u'text': u'five'},\n", - " {u'answer_start': 295,\n", - " u'text': u'five'}],\n", - " u'id': u'56d98b33dc89441400fdb53e',\n", - " u'question': u'How many tackles did Von Miller accomlish by himself in the game?'}]},\n", - " {u'context': u'CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyonc\\xe9 and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.',\n", - " u'qas': [{u'answers': [{u'answer_start': 0,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 0,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 0,\n", - " u'text': u'CBS'}],\n", - " u'id': u'56be5333acb8001400a5030a',\n", - " u'question': u'Which network broadcasted Super Bowl 50 in the U.S.?'},\n", - " {u'answers': [{u'answer_start': 67,\n", - " u'text': u'$5 million'},\n", - " {u'answer_start': 67,\n", - " u'text': u'$5 million'},\n", - " {u'answer_start': 67,\n", - " u'text': u'$5 million'}],\n", - " u'id': u'56be5333acb8001400a5030b',\n", - " u'question': u'What was the average cost for a 30 second commercial during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 194,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'}],\n", - " u'id': u'56be5333acb8001400a5030c',\n", - " u'question': u'Which group headlined the Super Bowl 50 halftime show?'},\n", - " {u'answers': [{u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'},\n", - " {u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'},\n", - " {u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'}],\n", - " u'id': u'56be5333acb8001400a5030d',\n", - " u'question': u'Which performers joined the headliner during the Super Bowl 50 halftime show?'},\n", - " {u'answers': [{u'answer_start': 275,\n", - " u'text': u'Super Bowl XLVII'},\n", - " {u'answer_start': 275,\n", - " u'text': u'Super Bowl XLVII'},\n", - " {u'answer_start': 286,\n", - " u'text': u'XLVII'}],\n", - " u'id': u'56be5333acb8001400a5030e',\n", - " u'question': u'At which Super Bowl did Beyonce headline the halftime show?'},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 0,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 0,\n", - " u'text': u'CBS'}],\n", - " u'id': u'56beaf5e3aeaaa14008c91fd',\n", - " u'question': u'Who was the broadcaster for Super Bowl 50 in the United States?'},\n", - " {u'answers': [{u'answer_start': 67,\n", - " u'text': u'$5 million'},\n", - " {u'answer_start': 67,\n", - " u'text': u'$5 million'},\n", - " {u'answer_start': 67,\n", - " u'text': u'$5 million'}],\n", - " u'id': u'56beaf5e3aeaaa14008c91fe',\n", - " u'question': u'What was the average cost of a 30-second commercial?'},\n", - " {u'answers': [{u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9'},\n", - " {u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9'},\n", - " {u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9'}],\n", - " u'id': u'56beaf5e3aeaaa14008c91ff',\n", - " u'question': u'What halftime performer previously headlined Super Bowl XLVII?'},\n", - " {u'answers': [{u'answer_start': 245,\n", - " u'text': u'Bruno Mars'},\n", - " {u'answer_start': 245,\n", - " u'text': u'Bruno Mars'},\n", - " {u'answer_start': 251,\n", - " u'text': u'Mars'}],\n", - " u'id': u'56beaf5e3aeaaa14008c9200',\n", - " u'question': u'What halftime performer previously headlined Super Bowl XLVIII?'},\n", - " {u'answers': [{u'answer_start': 194,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'}],\n", - " u'id': u'56beaf5e3aeaaa14008c9201',\n", - " u'question': u\"Who was the main performer at this year's halftime show?\"},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 0,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 0,\n", - " u'text': u'CBS'}],\n", - " u'id': u'56bf1ae93aeaaa14008c951b',\n", - " u'question': u'Which network broadcasted the 50th Super Bowl game? '},\n", - " {u'answers': [{u'answer_start': 67,\n", - " u'text': u'$5 million'},\n", - " {u'answer_start': 67,\n", - " u'text': u'$5 million'},\n", - " {u'answer_start': 67,\n", - " u'text': u'$5 million'}],\n", - " u'id': u'56bf1ae93aeaaa14008c951c',\n", - " u'question': u'What was the average cost for a TV ad lasting 30 seconds during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 245,\n", - " u'text': u'Bruno Mars'},\n", - " {u'answer_start': 245,\n", - " u'text': u'Bruno Mars'},\n", - " {u'answer_start': 245,\n", - " u'text': u'Bruno Mars,'}],\n", - " u'id': u'56bf1ae93aeaaa14008c951e',\n", - " u'question': u'Who was the male singer who performed as a special guest during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 355,\n", - " u'text': u'third'},\n", - " {u'answer_start': 355,\n", - " u'text': u'third'},\n", - " {u'answer_start': 355,\n", - " u'text': u'third'}],\n", - " u'id': u'56bf1ae93aeaaa14008c951f',\n", - " u'question': u'What ranking does the Super Bowl 50 halftime show have on the list of most watched TV broadcasts?'},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 0,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 0,\n", - " u'text': u'CBS'}],\n", - " u'id': u'56d2051ce7d4791d00902608',\n", - " u'question': u'What station aired the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 67,\n", - " u'text': u'$5 million'},\n", - " {u'answer_start': 67,\n", - " u'text': u'$5 million'},\n", - " {u'answer_start': 67,\n", - " u'text': u'$5 million'}],\n", - " u'id': u'56d2051ce7d4791d00902609',\n", - " u'question': u'How much money did a 1/2 minute commercial cost?'},\n", - " {u'answers': [{u'answer_start': 194,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'}],\n", - " u'id': u'56d2051ce7d4791d0090260a',\n", - " u'question': u'What band headlined half-time during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'},\n", - " {u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'},\n", - " {u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'}],\n", - " u'id': u'56d2051ce7d4791d0090260b',\n", - " u'question': u'What two artists came out with Coldplay during the half-time show?'},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 0,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 0,\n", - " u'text': u'CBS'}],\n", - " u'id': u'56d602631c85041400946ed8',\n", - " u'question': u'Who broadcast the Super Bowl on TV?'},\n", - " {u'answers': [{u'answer_start': 194,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'}],\n", - " u'id': u'56d602631c85041400946eda',\n", - " u'question': u'Who headlined the halftime show for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'},\n", - " {u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'},\n", - " {u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'}],\n", - " u'id': u'56d602631c85041400946edb',\n", - " u'question': u'Who were special guests for the Super Bowl halftime show?'},\n", - " {u'answers': [{u'answer_start': 275,\n", - " u'text': u'Super Bowl XLVII'},\n", - " {u'answer_start': 275,\n", - " u'text': u'Super Bowl XLVII'},\n", - " {u'answer_start': 275,\n", - " u'text': u'Super Bowl XLVII'}],\n", - " u'id': u'56d602631c85041400946edc',\n", - " u'question': u'Which Super Bowl halftime show did Beyonc\\xe9 headline?'},\n", - " {u'answers': [{u'answer_start': 67,\n", - " u'text': u'$5 million'},\n", - " {u'answer_start': 67,\n", - " u'text': u'$5 million'},\n", - " {u'answer_start': 67,\n", - " u'text': u'$5 million for a 30-second'}],\n", - " u'id': u'56d98c53dc89441400fdb544',\n", - " u'question': u'What was the cost for a half minute ad?'},\n", - " {u'answers': [{u'answer_start': 194,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'}],\n", - " u'id': u'56d98c53dc89441400fdb545',\n", - " u'question': u'Who lead the Super Bowl 50 halftime performance?'},\n", - " {u'answers': [{u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'},\n", - " {u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'},\n", - " {u'answer_start': 233,\n", - " u'text': u'Beyonc\\xe9 and Bruno Mars'}],\n", - " u'id': u'56d98c53dc89441400fdb546',\n", - " u'question': u'What other two famous performers were part of the Super Bowl 50 halftime?'},\n", - " {u'answers': [{u'answer_start': 245,\n", - " u'text': u'Bruno Mars'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 194,\n", - " u'text': u'Coldplay'}],\n", - " u'id': u'56d98c53dc89441400fdb548',\n", - " u'question': u'What performer lead the Super Bowl XLVIII halftime show?'}]},\n", - " {u'context': u'In early 2012, NFL Commissioner Roger Goodell stated that the league planned to make the 50th Super Bowl \"spectacular\" and that it would be \"an important game for us as a league\".',\n", - " u'qas': [{u'answers': [{u'answer_start': 32,\n", - " u'text': u'Roger Goodell'},\n", - " {u'answer_start': 32,\n", - " u'text': u'Roger Goodell'},\n", - " {u'answer_start': 38,\n", - " u'text': u'Goodell'}],\n", - " u'id': u'56be53b8acb8001400a50314',\n", - " u'question': u'Who was the NFL Commissioner in early 2012?'},\n", - " {u'answers': [{u'answer_start': 85,\n", - " u'text': u'the 50th Super Bowl'},\n", - " {u'answer_start': 85,\n", - " u'text': u'the 50th'},\n", - " {u'answer_start': 89,\n", - " u'text': u'50th'}],\n", - " u'id': u'56be53b8acb8001400a50315',\n", - " u'question': u'Which Super Bowl did Roger Goodell speak about?'},\n", - " {u'answers': [{u'answer_start': 9,\n", - " u'text': u'2012'},\n", - " {u'answer_start': 9,\n", - " u'text': u'2012'},\n", - " {u'answer_start': 9,\n", - " u'text': u'2012'}],\n", - " u'id': u'56be53b8acb8001400a50316',\n", - " u'question': u\"In what year did Roger Goodell call Super Bowl 50 'an important game for us as a league'?\"},\n", - " {u'answers': [{u'answer_start': 32,\n", - " u'text': u'Roger Goodell'},\n", - " {u'answer_start': 32,\n", - " u'text': u'Roger Goodell'},\n", - " {u'answer_start': 38,\n", - " u'text': u'Goodell'}],\n", - " u'id': u'56beafca3aeaaa14008c9207',\n", - " u'question': u'Who is the Commissioner of the National Football League?'},\n", - " {u'answers': [{u'answer_start': 3,\n", - " u'text': u'early 2012'},\n", - " {u'answer_start': 0,\n", - " u'text': u'In early 2012'},\n", - " {u'answer_start': 9,\n", - " u'text': u'2012'}],\n", - " u'id': u'56beafca3aeaaa14008c9208',\n", - " u'question': u'When did he make the quoted remarks about Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 32,\n", - " u'text': u'Roger Goodell'},\n", - " {u'answer_start': 32,\n", - " u'text': u'Roger Goodell'},\n", - " {u'answer_start': 38,\n", - " u'text': u'Goodell'}],\n", - " u'id': u'56bf42f53aeaaa14008c95a3',\n", - " u'question': u'Who was the commissioner of the NFL in 2012? '},\n", - " {u'answers': [{u'answer_start': 32,\n", - " u'text': u'Roger Goodell'},\n", - " {u'answer_start': 32,\n", - " u'text': u'Roger Goodell'},\n", - " {u'answer_start': 38,\n", - " u'text': u'Goodell'}],\n", - " u'id': u'56d2053ae7d4791d00902610',\n", - " u'question': u'Who if the commissioner of the NFL?'},\n", - " {u'answers': [{u'answer_start': 32,\n", - " u'text': u'Roger Goodell'},\n", - " {u'answer_start': 32,\n", - " u'text': u'Roger Goodell'},\n", - " {u'answer_start': 38,\n", - " u'text': u'Goodell'}],\n", - " u'id': u'56d6edd00d65d21400198250',\n", - " u'question': u'Who is the commissioner of the NFL?'},\n", - " {u'answers': [{u'answer_start': 106,\n", - " u'text': u'spectacular'},\n", - " {u'answer_start': 141,\n", - " u'text': u'an important game for us as a league'},\n", - " {u'answer_start': 106,\n", - " u'text': u'spectacular'}],\n", - " u'id': u'56d6edd00d65d21400198251',\n", - " u'question': u'In early 2012, Goodell said that Super Bowl 50 would be what?'},\n", - " {u'answers': [{u'answer_start': 106,\n", - " u'text': u'spectacular'},\n", - " {u'answer_start': 106,\n", - " u'text': u'spectacular'},\n", - " {u'answer_start': 106,\n", - " u'text': u'spectacular'}],\n", - " u'id': u'56d98d0adc89441400fdb54e',\n", - " u'question': u'What one word did the NFL commissioner use to describe what Super Bowl 50 was intended to be?'},\n", - " {u'answers': [{u'answer_start': 9,\n", - " u'text': u'2012'},\n", - " {u'answer_start': 9,\n", - " u'text': u'2012'},\n", - " {u'answer_start': 9,\n", - " u'text': u'2012'}],\n", - " u'id': u'56d98d0adc89441400fdb54f',\n", - " u'question': u'What year did Roger Goodell announce that Super Bowl 50 would be \"important\"?'}]},\n", - " {u'context': u\"The league eventually narrowed the bids to three sites: New Orleans' Mercedes-Benz Superdome, Miami's Sun Life Stadium, and the San Francisco Bay Area's Levi's Stadium.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 56,\n", - " u'text': u\"New Orleans' Mercedes-Benz Superdome\"},\n", - " {u'answer_start': 56,\n", - " u'text': u\"New Orleans' Mercedes-Benz Superdome\"},\n", - " {u'answer_start': 69,\n", - " u'text': u'Mercedes-Benz Superdome'}],\n", - " u'id': u'56be5438acb8001400a5031a',\n", - " u'question': u'Which Louisiana venue was one of three considered for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 94,\n", - " u'text': u\"Miami's Sun Life Stadium\"},\n", - " {u'answer_start': 94,\n", - " u'text': u\"Miami's Sun Life Stadium\"},\n", - " {u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'}],\n", - " u'id': u'56be5438acb8001400a5031b',\n", - " u'question': u'Which Florida venue was one of three considered for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 128,\n", - " u'text': u\"San Francisco Bay Area's Levi's Stadium\"},\n", - " {u'answer_start': 128,\n", - " u'text': u\"San Francisco Bay Area's Levi's Stadium\"},\n", - " {u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium\"}],\n", - " u'id': u'56be5438acb8001400a5031c',\n", - " u'question': u'Which California venue was one of three considered for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'},\n", - " {u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'},\n", - " {u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'}],\n", - " u'id': u'56beb03c3aeaaa14008c920b',\n", - " u'question': u'What venue in Miami was a candidate for the site of Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium\"}],\n", - " u'id': u'56beb03c3aeaaa14008c920d',\n", - " u'question': u'What site is located in the San Francisco Bay Area?'},\n", - " {u'answers': [{u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium.\"}],\n", - " u'id': u'56bf3c633aeaaa14008c9580',\n", - " u'question': u\"What is the name of San Francisco's stadium when looked at as a possibility for Super Bowl 50?\"},\n", - " {u'answers': [{u'answer_start': 69,\n", - " u'text': u'Mercedes-Benz Superdome'},\n", - " {u'answer_start': 69,\n", - " u'text': u'Mercedes-Benz Superdome'},\n", - " {u'answer_start': 69,\n", - " u'text': u'Mercedes-Benz Superdome'}],\n", - " u'id': u'56bf3c633aeaaa14008c9581',\n", - " u'question': u\"What was the name of New Orleans' superdome at the time that Super Bowl 50 took place?\"},\n", - " {u'answers': [{u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'},\n", - " {u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'},\n", - " {u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'}],\n", - " u'id': u'56bf3c633aeaaa14008c9582',\n", - " u'question': u\"What was the given name of Miami's stadium at the time of Super Bowl 50?\"},\n", - " {u'answers': [{u'answer_start': 56,\n", - " u'text': u\"New Orleans' Mercedes-Benz Superdome, Miami's Sun Life Stadium, and the San Francisco Bay Area's Levi's Stadium\"},\n", - " {u'answer_start': 56,\n", - " u'text': u\"New Orleans' Mercedes-Benz Superdome, Miami's Sun Life Stadium, and the San Francisco Bay Area's Levi's Stadium.\"},\n", - " {u'answer_start': 56,\n", - " u'text': u\"New Orleans' Mercedes-Benz Superdome, Miami's Sun Life Stadium, and the San Francisco Bay Area's Levi's Stadium.\"}],\n", - " u'id': u'56d20564e7d4791d00902612',\n", - " u'question': u'What three stadiums did the NFL decide between for the game?'},\n", - " {u'answers': [{u'answer_start': 43,\n", - " u'text': u'three'},\n", - " {u'answer_start': 43,\n", - " u'text': u'three'},\n", - " {u'answer_start': 43,\n", - " u'text': u'three'}],\n", - " u'id': u'56d6ee6e0d65d21400198254',\n", - " u'question': u\"How many sites did the NFL narrow down Super Bowl 50's location to?\"},\n", - " {u'answers': [{u'answer_start': 56,\n", - " u'text': u'New Orleans'},\n", - " {u'answer_start': 56,\n", - " u'text': u'New Orleans'},\n", - " {u'answer_start': 56,\n", - " u'text': u\"New Orleans'\"}],\n", - " u'id': u'56d6ee6e0d65d21400198255',\n", - " u'question': u'One of the sites, Merceds-Benz Superdome, is located where?'},\n", - " {u'answers': [{u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'},\n", - " {u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'},\n", - " {u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'}],\n", - " u'id': u'56d6ee6e0d65d21400198256',\n", - " u'question': u'What is the name of the stadium in Miami that was considered?'},\n", - " {u'answers': [{u'answer_start': 128,\n", - " u'text': u'San Francisco'},\n", - " {u'answer_start': 128,\n", - " u'text': u'San Francisco'},\n", - " {u'answer_start': 128,\n", - " u'text': u\"San Francisco Bay Area's\"}],\n", - " u'id': u'56d6ee6e0d65d21400198257',\n", - " u'question': u'What was the third city that was considered?'},\n", - " {u'answers': [{u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium.\"},\n", - " {u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium.\"}],\n", - " u'id': u'56d6ee6e0d65d21400198258',\n", - " u'question': u'What is the name of the stadium in San Francisco Bay Area?'},\n", - " {u'answers': [{u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'},\n", - " {u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'},\n", - " {u'answer_start': 102,\n", - " u'text': u'Sun Life Stadium'}],\n", - " u'id': u'56d98db6dc89441400fdb552',\n", - " u'question': u'What Florida stadium was considered for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 69,\n", - " u'text': u'Mercedes-Benz Superdome'},\n", - " {u'answer_start': 69,\n", - " u'text': u'Mercedes-Benz Superdome'},\n", - " {u'answer_start': 69,\n", - " u'text': u'Mercedes-Benz Superdome,'}],\n", - " u'id': u'56d98db6dc89441400fdb553',\n", - " u'question': u'What New Orleans stadium was considered for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium.\"},\n", - " {u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium\"},\n", - " {u'answer_start': 153,\n", - " u'text': u\"Levi's Stadium.\"}],\n", - " u'id': u'56d98db6dc89441400fdb554',\n", - " u'question': u'What is the name of the stadium where Super Bowl 50 was played?'}]},\n", - " {u'context': u\"The league announced on October 16, 2012, that the two finalists were Sun Life Stadium and Levi's Stadium. The South Florida/Miami area has previously hosted the event 10 times (tied for most with New Orleans), with the most recent one being Super Bowl XLIV in 2010. The San Francisco Bay Area last hosted in 1985 (Super Bowl XIX), held at Stanford Stadium in Stanford, California, won by the home team 49ers. The Miami bid depended on whether the stadium underwent renovations. However, on May 3, 2013, the Florida legislature refused to approve the funding plan to pay for the renovations, dealing a significant blow to Miami's chances.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 24,\n", - " u'text': u'October 16, 2012'},\n", - " {u'answer_start': 24,\n", - " u'text': u'October 16, 2012,'},\n", - " {u'answer_start': 24,\n", - " u'text': u'October 16, 2012'}],\n", - " u'id': u'56be54bdacb8001400a50322',\n", - " u'question': u'When were the two finalists for hosting Super Bowl 50 announced?'},\n", - " {u'answers': [{u'answer_start': 168,\n", - " u'text': u'10'},\n", - " {u'answer_start': 168,\n", - " u'text': u'10'},\n", - " {u'answer_start': 168,\n", - " u'text': u'10'}],\n", - " u'id': u'56be54bdacb8001400a50323',\n", - " u'question': u'How many times has the South Florida/Miami area hosted the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 242,\n", - " u'text': u'Super Bowl XLIV'},\n", - " {u'answer_start': 242,\n", - " u'text': u'Super Bowl XLIV'},\n", - " {u'answer_start': 261,\n", - " u'text': u'2010'}],\n", - " u'id': u'56be54bdacb8001400a50324',\n", - " u'question': u'What was the most recent Super Bowl hosted in the South Florida/Miami area?'},\n", - " {u'answers': [{u'answer_start': 261,\n", - " u'text': u'2010'},\n", - " {u'answer_start': 261,\n", - " u'text': u'2010'},\n", - " {u'answer_start': 261,\n", - " u'text': u'2010'}],\n", - " u'id': u'56be54bdacb8001400a50325',\n", - " u'question': u'When was the most recent Super Bowl hosted in the South Florida/Miami area?'},\n", - " {u'answers': [{u'answer_start': 309,\n", - " u'text': u'1985'},\n", - " {u'answer_start': 309,\n", - " u'text': u'1985'},\n", - " {u'answer_start': 309,\n", - " u'text': u'1985'}],\n", - " u'id': u'56be54bdacb8001400a50326',\n", - " u'question': u'When did the San Francisco Bay area last host the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 70,\n", - " u'text': u'Sun Life Stadium'},\n", - " {u'answer_start': 70,\n", - " u'text': u'Sun Life Stadium'},\n", - " {u'answer_start': 70,\n", - " u'text': u'Sun Life Stadium'}],\n", - " u'id': u'56beb0f43aeaaa14008c921b',\n", - " u'question': u\"What was the other finalist besides Levi's Stadium?\"},\n", - " {u'answers': [{u'answer_start': 24,\n", - " u'text': u'October 16, 2012'},\n", - " {u'answer_start': 24,\n", - " u'text': u'October 16, 2012'},\n", - " {u'answer_start': 24,\n", - " u'text': u'October 16, 2012,'}],\n", - " u'id': u'56beb0f43aeaaa14008c921c',\n", - " u'question': u'When were the finalists announced?'},\n", - " {u'answers': [{u'answer_start': 340,\n", - " u'text': u'Stanford Stadium'},\n", - " {u'answer_start': 340,\n", - " u'text': u'Stanford Stadium'},\n", - " {u'answer_start': 340,\n", - " u'text': u'Stanford Stadium'}],\n", - " u'id': u'56beb0f43aeaaa14008c921d',\n", - " u'question': u'In what venue did Super Bowl XIX take place?'},\n", - " {u'answers': [{u'answer_start': 491,\n", - " u'text': u'May 3, 2013'},\n", - " {u'answer_start': 491,\n", - " u'text': u'May 3, 2013'},\n", - " {u'answer_start': 491,\n", - " u'text': u'May 3, 2013'}],\n", - " u'id': u'56beb0f43aeaaa14008c921e',\n", - " u'question': u'On what date did the Florida legislature decide against the plan to renovate the Miami stadium?'},\n", - " {u'answers': [{u'answer_start': 261,\n", - " u'text': u'2010'},\n", - " {u'answer_start': 261,\n", - " u'text': u'2010'},\n", - " {u'answer_start': 261,\n", - " u'text': u'2010'}],\n", - " u'id': u'56beb0f43aeaaa14008c921f',\n", - " u'question': u'In what year was the Super Bowl last held in the Miami/South Florida area?'},\n", - " {u'answers': [{u'answer_start': 51,\n", - " u'text': u'two'},\n", - " {u'answer_start': 168,\n", - " u'text': u'10'},\n", - " {u'answer_start': 168,\n", - " u'text': u'10'}],\n", - " u'id': u'56bf21b43aeaaa14008c9525',\n", - " u'question': u\"How many times has a Super Bowl taken place at Miami's Sun Life Stadium?\"},\n", - " {u'answers': [{u'answer_start': 242,\n", - " u'text': u'Super Bowl XLIV'},\n", - " {u'answer_start': 242,\n", - " u'text': u'Super Bowl XLIV'},\n", - " {u'answer_start': 261,\n", - " u'text': u'2010'}],\n", - " u'id': u'56bf21b43aeaaa14008c9526',\n", - " u'question': u'What was the last Super Bowl that took place at Sun Life Stadium in Miami? '},\n", - " {u'answers': [{u'answer_start': 51,\n", - " u'text': u'two'},\n", - " {u'answer_start': 51,\n", - " u'text': u'two'},\n", - " {u'answer_start': 51,\n", - " u'text': u'two'}],\n", - " u'id': u'56bf21b43aeaaa14008c9528',\n", - " u'question': u'In 2012, how many stadiums were named as finalists for hosting Super Bowl 50 before the final stadium was chosen?'},\n", - " {u'answers': [{u'answer_start': 508,\n", - " u'text': u'Florida legislature'},\n", - " {u'answer_start': 504,\n", - " u'text': u'the Florida legislature'},\n", - " {u'answer_start': 508,\n", - " u'text': u'Florida legislature'}],\n", - " u'id': u'56bf21b43aeaaa14008c9529',\n", - " u'question': u\"What was the entity that stepped in and caused Miami's Sun Life Stadium to no longer be in the running to host Super Bowl 50?\"},\n", - " {u'answers': [{u'answer_start': 309,\n", - " u'text': u'1985'},\n", - " {u'answer_start': 309,\n", - " u'text': u'1985'},\n", - " {u'answer_start': 309,\n", - " u'text': u'1985'}],\n", - " u'id': u'56d6ef6a0d65d21400198260',\n", - " u'question': u'Prior to this consideration, when did San Francisco last host a Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 197,\n", - " u'text': u'New Orleans'},\n", - " {u'answer_start': 197,\n", - " u'text': u'New Orleans'},\n", - " {u'answer_start': 197,\n", - " u'text': u'New Orleans'}],\n", - " u'id': u'56d6ef6a0d65d21400198262',\n", - " u'question': u'What other city has hosted the Super Bowl ten times?'},\n", - " {u'answers': [{u'answer_start': 24,\n", - " u'text': u'October 16, 2012'},\n", - " {u'answer_start': 24,\n", - " u'text': u'October 16, 2012'},\n", - " {u'answer_start': 24,\n", - " u'text': u'October 16, 2012,'}],\n", - " u'id': u'56d98f0ddc89441400fdb558',\n", - " u'question': u'What date were the top two stadium choices for Super Bowl 50 announced?'},\n", - " {u'answers': [{u'answer_start': 263,\n", - " u'text': u'10.'},\n", - " {u'answer_start': 168,\n", - " u'text': u'10'},\n", - " {u'answer_start': 168,\n", - " u'text': u'10 times'}],\n", - " u'id': u'56d98f0ddc89441400fdb559',\n", - " u'question': u'How many times prios has the Sun Life Stadium had Super Bowls?'},\n", - " {u'answers': [{u'answer_start': 197,\n", - " u'text': u'New Orleans'},\n", - " {u'answer_start': 197,\n", - " u'text': u'New Orleans'},\n", - " {u'answer_start': 197,\n", - " u'text': u'New Orleans'}],\n", - " u'id': u'56d98f0ddc89441400fdb55a',\n", - " u'question': u'What city is tied with Miami for hosting the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 309,\n", - " u'text': u'1985'},\n", - " {u'answer_start': 309,\n", - " u'text': u'1985'},\n", - " {u'answer_start': 309,\n", - " u'text': u'1985'}],\n", - " u'id': u'56d98f0ddc89441400fdb55b',\n", - " u'question': u'When was the last time San Francisco hosted a Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 508,\n", - " u'text': u'Florida legislature'},\n", - " {u'answer_start': 504,\n", - " u'text': u'the Florida legislature'},\n", - " {u'answer_start': 508,\n", - " u'text': u'Florida legislature'}],\n", - " u'id': u'56d98f0ddc89441400fdb55c',\n", - " u'question': u'Who decided not to approve paying for renovations at Sun Life Stadium that the league wanted for them to do to host Super Bowl 50?'}]},\n", - " {u'context': u\"On May 21, 2013, NFL owners at their spring meetings in Boston voted and awarded the game to Levi's Stadium. The $1.2 billion stadium opened in 2014. It is the first Super Bowl held in the San Francisco Bay Area since Super Bowl XIX in 1985, and the first in California since Super Bowl XXXVII took place in San Diego in 2003.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 3,\n", - " u'text': u'May 21, 2013'},\n", - " {u'answer_start': 3,\n", - " u'text': u'May 21, 2013'},\n", - " {u'answer_start': 3,\n", - " u'text': u'May 21, 2013,'}],\n", - " u'id': u'56be5523acb8001400a5032c',\n", - " u'question': u\"When was Levi's Stadium awarded the right to host Super Bowl 50?\"},\n", - " {u'answers': [{u'answer_start': 17,\n", - " u'text': u'NFL owners'},\n", - " {u'answer_start': 17,\n", - " u'text': u'NFL owners'},\n", - " {u'answer_start': 17,\n", - " u'text': u'NFL owners'}],\n", - " u'id': u'56be5523acb8001400a5032d',\n", - " u'question': u'Who voted on the venue for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 144,\n", - " u'text': u'2014'},\n", - " {u'answer_start': 141,\n", - " u'text': u'in 2014'},\n", - " {u'answer_start': 144,\n", - " u'text': u'2014'}],\n", - " u'id': u'56be5523acb8001400a5032e',\n", - " u'question': u\"When did Lev's Stadium open?\"},\n", - " {u'answers': [{u'answer_start': 113,\n", - " u'text': u'$1.2 billion'},\n", - " {u'answer_start': 113,\n", - " u'text': u'$1.2 billion'},\n", - " {u'answer_start': 113,\n", - " u'text': u'$1.2 billion'}],\n", - " u'id': u'56be5523acb8001400a5032f',\n", - " u'question': u\"How much did it cost to build Levi's Stadium?\"},\n", - " {u'answers': [{u'answer_start': 308,\n", - " u'text': u'San Diego'},\n", - " {u'answer_start': 308,\n", - " u'text': u'San Diego'},\n", - " {u'answer_start': 308,\n", - " u'text': u'San Diego'}],\n", - " u'id': u'56be5523acb8001400a50330',\n", - " u'question': u'What California city last hosted the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 56,\n", - " u'text': u'Boston'},\n", - " {u'answer_start': 53,\n", - " u'text': u'in Boston'},\n", - " {u'answer_start': 3,\n", - " u'text': u'May 21, 2013'}],\n", - " u'id': u'56beb2153aeaaa14008c9225',\n", - " u'question': u'Where did the spring meetings of the NFL owners take place?'},\n", - " {u'answers': [{u'answer_start': 3,\n", - " u'text': u'May 21, 2013'},\n", - " {u'answer_start': 3,\n", - " u'text': u'May 21, 2013'},\n", - " {u'answer_start': 3,\n", - " u'text': u'May 21, 2013,'}],\n", - " u'id': u'56beb2153aeaaa14008c9226',\n", - " u'question': u\"On what date was Super Bowl 50 given to Levi's Stadium?\"},\n", - " {u'answers': [{u'answer_start': 113,\n", - " u'text': u'$1.2 billion'},\n", - " {u'answer_start': 113,\n", - " u'text': u'$1.2 billion'},\n", - " {u'answer_start': 113,\n", - " u'text': u'$1.2 billion'}],\n", - " u'id': u'56beb2153aeaaa14008c9227',\n", - " u'question': u\"How much did it cost to build Levi's Stadium?\"},\n", - " {u'answers': [{u'answer_start': 276,\n", - " u'text': u'Super Bowl XXXVII'},\n", - " {u'answer_start': 276,\n", - " u'text': u'Super Bowl XXXVII'},\n", - " {u'answer_start': 287,\n", - " u'text': u'XXXVII'}],\n", - " u'id': u'56beb2153aeaaa14008c9228',\n", - " u'question': u'Prior to Super Bowl 50, what was the last Super Bowl in California?'},\n", - " {u'answers': [{u'answer_start': 308,\n", - " u'text': u'San Diego'},\n", - " {u'answer_start': 308,\n", - " u'text': u'San Diego'},\n", - " {u'answer_start': 308,\n", - " u'text': u'San Diego'}],\n", - " u'id': u'56beb2153aeaaa14008c9229',\n", - " u'question': u'In what city did the last Super Bowl in California occur?'},\n", - " {u'answers': [{u'answer_start': 11,\n", - " u'text': u'2013'},\n", - " {u'answer_start': 11,\n", - " u'text': u'2013'},\n", - " {u'answer_start': 11,\n", - " u'text': u'2013'}],\n", - " u'id': u'56bf23363aeaaa14008c952f',\n", - " u'question': u\"What year did Levi's Stadium become fully approved to host Super Bowl 50?\"},\n", - " {u'answers': [{u'answer_start': 144,\n", - " u'text': u'2014'},\n", - " {u'answer_start': 144,\n", - " u'text': u'2014'},\n", - " {u'answer_start': 144,\n", - " u'text': u'2014'}],\n", - " u'id': u'56bf23363aeaaa14008c9530',\n", - " u'question': u\"When did Levi's stadium open to the public? \"},\n", - " {u'answers': [{u'answer_start': 113,\n", - " u'text': u'$1.2 billion'},\n", - " {u'answer_start': 113,\n", - " u'text': u'$1.2 billion'},\n", - " {u'answer_start': 113,\n", - " u'text': u'$1.2 billion'}],\n", - " u'id': u'56bf23363aeaaa14008c9531',\n", - " u'question': u'How much did it cost to build the stadium where Super Bowl 50 was played?'},\n", - " {u'answers': [{u'answer_start': 236,\n", - " u'text': u'1985'},\n", - " {u'answer_start': 236,\n", - " u'text': u'1985'},\n", - " {u'answer_start': 236,\n", - " u'text': u'1985'}],\n", - " u'id': u'56bf23363aeaaa14008c9532',\n", - " u'question': u'What year did a Super Bowl play in the bay area around San Francisco, prior to Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 276,\n", - " u'text': u'Super Bowl XXXVII'},\n", - " {u'answer_start': 276,\n", - " u'text': u'Super Bowl XXXVII'},\n", - " {u'answer_start': 287,\n", - " u'text': u'XXXVII'}],\n", - " u'id': u'56bf23363aeaaa14008c9533',\n", - " u'question': u'Which Super Bowl was hosted in San Diego in 2003? '},\n", - " {u'answers': [{u'answer_start': 3,\n", - " u'text': u'May 21, 2013'},\n", - " {u'answer_start': 3,\n", - " u'text': u'May 21, 2013,'},\n", - " {u'answer_start': 3,\n", - " u'text': u'May 21, 2013'}],\n", - " u'id': u'56d6f0770d65d21400198268',\n", - " u'question': u'When was San Francisco voted to be the location for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 144,\n", - " u'text': u'2014'},\n", - " {u'answer_start': 141,\n", - " u'text': u'in 2014'},\n", - " {u'answer_start': 144,\n", - " u'text': u'2014'}],\n", - " u'id': u'56d6f0770d65d21400198269',\n", - " u'question': u\"When did Levi's Stadium open?\"},\n", - " {u'answers': [{u'answer_start': 321,\n", - " u'text': u'2003'},\n", - " {u'answer_start': 318,\n", - " u'text': u'in 2003'},\n", - " {u'answer_start': 321,\n", - " u'text': u'2003'}],\n", - " u'id': u'56d6f0770d65d2140019826a',\n", - " u'question': u'When was the last Super Bowl in California?'},\n", - " {u'answers': [{u'answer_start': 56,\n", - " u'text': u'Boston'},\n", - " {u'answer_start': 53,\n", - " u'text': u'in Boston'},\n", - " {u'answer_start': 56,\n", - " u'text': u'Boston'}],\n", - " u'id': u'56d6f0770d65d2140019826c',\n", - " u'question': u'Where was the meeting held when the NFL owners voted on the location for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 3,\n", - " u'text': u'May 21, 2013'},\n", - " {u'answer_start': 3,\n", - " u'text': u'May 21, 2013'},\n", - " {u'answer_start': 3,\n", - " u'text': u'May 21, 2013'}],\n", - " u'id': u'56d98fbfdc89441400fdb562',\n", - " u'question': u\"When was Levi's Stadium picked for Super bowl 50?\"},\n", - " {u'answers': [{u'answer_start': 144,\n", - " u'text': u'2014.'},\n", - " {u'answer_start': 141,\n", - " u'text': u'in 2014'},\n", - " {u'answer_start': 144,\n", - " u'text': u'2014'}],\n", - " u'id': u'56d98fbfdc89441400fdb563',\n", - " u'question': u\"When did Levi's Stadium open?\"},\n", - " {u'answers': [{u'answer_start': 113,\n", - " u'text': u'$1.2 billion'},\n", - " {u'answer_start': 113,\n", - " u'text': u'$1.2 billion'},\n", - " {u'answer_start': 113,\n", - " u'text': u'$1.2 billion'}],\n", - " u'id': u'56d98fbfdc89441400fdb564',\n", - " u'question': u\"How much did Levi's Stadium cost?\"},\n", - " {u'answers': [{u'answer_start': 321,\n", - " u'text': u'2003.'},\n", - " {u'answer_start': 321,\n", - " u'text': u'2003'},\n", - " {u'answer_start': 321,\n", - " u'text': u'2003'}],\n", - " u'id': u'56d98fbfdc89441400fdb565',\n", - " u'question': u'When was the last time California hosted a Super Bowl?'}]},\n", - " {u'context': u'For the third straight season, the number one seeds from both conferences met in the Super Bowl. The Carolina Panthers became one of only ten teams to have completed a regular season with only one loss, and one of only six teams to have acquired a 15\\u20131 record, while the Denver Broncos became one of four teams to have made eight appearances in the Super Bowl. The Broncos made their second Super Bowl appearance in three years, having reached Super Bowl XLVIII, while the Panthers made their second Super Bowl appearance in franchise history, their other appearance being Super Bowl XXXVIII. Coincidentally, both teams were coached by John Fox in their last Super Bowl appearance prior to Super Bowl 50.',\n", - " u'qas': [{u'answers': [{u'answer_start': 636,\n", - " u'text': u'John Fox'},\n", - " {u'answer_start': 636,\n", - " u'text': u'John Fox'},\n", - " {u'answer_start': 641,\n", - " u'text': u'Fox'}],\n", - " u'id': u'56be572b3aeaaa14008c9052',\n", - " u'question': u'Who coached each Super Bowl 50 participant in their most recent Super Bowl appearance prior to Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 138,\n", - " u'text': u'ten'},\n", - " {u'answer_start': 138,\n", - " u'text': u'ten'},\n", - " {u'answer_start': 219,\n", - " u'text': u'six'}],\n", - " u'id': u'56beb2a03aeaaa14008c922f',\n", - " u'question': u'How many NFL teams have finished the regular season with one loss?'},\n", - " {u'answers': [{u'answer_start': 219,\n", - " u'text': u'six'},\n", - " {u'answer_start': 219,\n", - " u'text': u'six'},\n", - " {u'answer_start': 219,\n", - " u'text': u'six'}],\n", - " u'id': u'56beb2a03aeaaa14008c9230',\n", - " u'question': u'How many NFL teams have gone 15-1 in one season?'},\n", - " {u'answers': [{u'answer_start': 101,\n", - " u'text': u'Carolina Panthers'},\n", - " {u'answer_start': 97,\n", - " u'text': u'The Carolina Panthers'},\n", - " {u'answer_start': 110,\n", - " u'text': u'Panthers'}],\n", - " u'id': u'56beb2a03aeaaa14008c9231',\n", - " u'question': u'Which team in Super Bowl 50 had a 15-1 record?'},\n", - " {u'answers': [{u'answer_start': 444,\n", - " u'text': u'Super Bowl XLVIII'},\n", - " {u'answer_start': 444,\n", - " u'text': u'Super Bowl XLVIII'},\n", - " {u'answer_start': 455,\n", - " u'text': u'XLVIII'}],\n", - " u'id': u'56beb2a03aeaaa14008c9232',\n", - " u'question': u'What was the last Super Bowl the Broncos participated in?'},\n", - " {u'answers': [{u'answer_start': 636,\n", - " u'text': u'John Fox'},\n", - " {u'answer_start': 636,\n", - " u'text': u'John Fox'},\n", - " {u'answer_start': 641,\n", - " u'text': u'Fox'}],\n", - " u'id': u'56beb2a03aeaaa14008c9233',\n", - " u'question': u'Who was the head coach of the Broncos in Super Bowl XLVIII?'},\n", - " {u'answers': [{u'answer_start': 324,\n", - " u'text': u'eight'},\n", - " {u'answer_start': 324,\n", - " u'text': u'eight'},\n", - " {u'answer_start': 324,\n", - " u'text': u'eight'}],\n", - " u'id': u'56bf28c73aeaaa14008c9539',\n", - " u'question': u'What was the number of times the Denver Broncos played in a Super Bowl by the time they reached Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 138,\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " u'text': u'ten'},\n", - " {u'answer_start': 138,\n", - " u'text': u'ten'},\n", - " {u'answer_start': 138,\n", - " u'text': u'ten'}],\n", - " u'id': u'56bf28c73aeaaa14008c953a',\n", - " u'question': u'How many NFL teams have had only one loss by the end of a regular season?'},\n", - " {u'answers': [{u'answer_start': 573,\n", - " u'text': u'Super Bowl XXXVIII'},\n", - " {u'answer_start': 573,\n", - " u'text': u'Super Bowl XXXVIII'},\n", - " {u'answer_start': 584,\n", - " u'text': u'XXXVIII'}],\n", - " u'id': u'56bf28c73aeaaa14008c953c',\n", - " u'question': u'What was the first Super Bowl that the Carolina Panthers played in? '},\n", - " {u'answers': [{u'answer_start': 219,\n", - " u'text': u'six'},\n", - " {u'answer_start': 138,\n", - " u'text': u'ten'},\n", - " {u'answer_start': 138,\n", - " u'text': u'ten'}],\n", - " u'id': u'56bf28c73aeaaa14008c953d',\n", - " u'question': u'How many teams can boast a 15\\u20131 regular season record?'},\n", - " {u'answers': [{u'answer_start': 35,\n", - " u'text': u'number one'},\n", - " {u'answer_start': 35,\n", - " u'text': u'number one'},\n", - " {u'answer_start': 42,\n", - " u'text': u'one'}],\n", - " u'id': u'56d6f1190d65d21400198272',\n", - " u'question': u'What seed was the Carolina Panthers?'},\n", - " {u'answers': [{u'answer_start': 35,\n", - " u'text': u'number one'},\n", - " {u'answer_start': 35,\n", - " u'text': u'number one'},\n", - " {u'answer_start': 42,\n", - " u'text': u'one'}],\n", - " u'id': u'56d6f1190d65d21400198273',\n", - " u'question': u'What seed was the Denver Broncos?'},\n", - " {u'answers': [{u'answer_start': 444,\n", - " u'text': u'Super Bowl XLVIII'},\n", - " {u'answer_start': 444,\n", - " u'text': u'Super Bowl XLVIII'},\n", - " {u'answer_start': 444,\n", - " u'text': u'Super Bowl XLVIII'}],\n", - " u'id': u'56d6f1190d65d21400198274',\n", - " u'question': u'Prior to Super Bowl 50, when were the Broncos last there?'},\n", - " {u'answers': [{u'answer_start': 573,\n", - " u'text': u'Super Bowl XXXVIII.'},\n", - " {u'answer_start': 573,\n", - " u'text': u'Super Bowl XXXVIII'},\n", - " {u'answer_start': 573,\n", - " u'text': u'Super Bowl XXXVIII'}],\n", - " u'id': u'56d6f1190d65d21400198275',\n", - " u'question': u'Prior to Super Bowl 50, when were the Carolina Panthers last there?'},\n", - " {u'answers': [{u'answer_start': 219,\n", - " u'text': u'six'},\n", - " {u'answer_start': 138,\n", - " u'text': u'ten'},\n", - " {u'answer_start': 138,\n", - " u'text': u'ten'}],\n", - " u'id': u'56d6f1190d65d21400198276',\n", - " u'question': u'How many teams have had a 15-1 record for the regular season?'},\n", - " {u'answers': [{u'answer_start': 42,\n", - " u'text': u'one'},\n", - " {u'answer_start': 251,\n", - " u'text': u'1'},\n", - " {u'answer_start': 251,\n", - " u'text': u'1'}],\n", - " u'id': u'56d99179dc89441400fdb56c',\n", - " u'question': u'How many games did the Panthers lose in the regular season before Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 300,\n", - " u'text': u'four'},\n", - " {u'answer_start': 300,\n", - " u'text': u'four'},\n", - " {u'answer_start': 300,\n", - " u'text': u'four'}],\n", - " u'id': u'56d99179dc89441400fdb56d',\n", - " u'question': u'How many teams up to Super Bowl 50 have been to the championship game eight times?'},\n", - " {u'answers': [{u'answer_start': 636,\n", - " u'text': u'John Fox'},\n", - " {u'answer_start': 636,\n", - " u'text': u'John Fox'},\n", - " {u'answer_start': 636,\n", - " u'text': u'John Fox'}],\n", - " u'id': u'56d99179dc89441400fdb570',\n", - " u'question': u\"Before Super Bowl 50, what was the coach's name that coached both teams for their last Super Bowl appearances?\"}]},\n", - " {u'context': u'Despite waiving longtime running back DeAngelo Williams and losing top wide receiver Kelvin Benjamin to a torn ACL in the preseason, the Carolina Panthers had their best regular season in franchise history, becoming the seventh team to win at least 15 regular season games since the league expanded to a 16-game schedule in 1978. Carolina started the season 14\\u20130, not only setting franchise records for the best start and the longest single-season winning streak, but also posting the best start to a season by an NFC team in NFL history, breaking the 13\\u20130 record previously shared with the 2009 New Orleans Saints and the 2011 Green Bay Packers. With their NFC-best 15\\u20131 regular season record, the Panthers clinched home-field advantage throughout the NFC playoffs for the first time in franchise history. Ten players were selected to the Pro Bowl (the most in franchise history) along with eight All-Pro selections.',\n", - " u'qas': [{u'answers': [{u'answer_start': 38,\n", - " u'text': u'DeAngelo Williams'},\n", - " {u'answer_start': 38,\n", - " u'text': u'DeAngelo Williams'},\n", - " {u'answer_start': 47,\n", - " u'text': u'Williams'}],\n", - " u'id': u'56be59683aeaaa14008c9058',\n", - " u'question': u'Whic Carolina Panthers running back was waived?'},\n", - " {u'answers': [{u'answer_start': 85,\n", - " u'text': u'Kelvin Benjamin'},\n", - " {u'answer_start': 85,\n", - " u'text': u'Kelvin Benjamin'},\n", - " {u'answer_start': 92,\n", - " u'text': u'Benjamin'}],\n", - " u'id': u'56be59683aeaaa14008c9059',\n", - " u'question': u'Which Carolina Panthers wide receiver suffered a torn ACL before the season began?'},\n", - " {u'answers': [{u'answer_start': 326,\n", - " u'text': u'7'},\n", - " {u'answer_start': 220,\n", - " u'text': u'seventh'},\n", - " {u'answer_start': 220,\n", - " u'text': u'seventh'}],\n", - " u'id': u'56be59683aeaaa14008c905a',\n", - " u'question': u'How many teams have won 15 regular season games since the 16-game schedule was adopted?'},\n", - " {u'answers': [{u'answer_start': 324,\n", - " u'text': u'1978'},\n", - " {u'answer_start': 324,\n", - " u'text': u'1978'},\n", - " {u'answer_start': 324,\n", - " u'text': u'1978'}],\n", - " u'id': u'56beb3083aeaaa14008c923d',\n", - " u'question': u'In what year did the NFL switch to a 16-game regular season?'},\n", - " {u'answers': [{u'answer_start': 137,\n", - " u'text': u'Carolina Panthers'},\n", - " {u'answer_start': 695,\n", - " u'text': u'the Panthers'},\n", - " {u'answer_start': 330,\n", - " u'text': u'Carolina'}],\n", - " u'id': u'56beb3083aeaaa14008c923e',\n", - " u'question': u'Who had the best record in the NFC?'},\n", - " {u'answers': [{u'answer_start': 807,\n", - " u'text': u'Ten'},\n", - " {u'answer_start': 807,\n", - " u'text': u'Ten'},\n", - " {u'answer_start': 807,\n", - " u'text': u'Ten'}],\n", - " u'id': u'56beb3083aeaaa14008c923f',\n", - " u'question': u'How many Panthers went to the Pro Bowl?'},\n", - " {u'answers': [{u'answer_start': 892,\n", - " u'text': u'eight'},\n", - " {u'answer_start': 892,\n", - " u'text': u'eight'},\n", - " {u'answer_start': 892,\n", - " u'text': u'eight'}],\n", - " u'id': u'56beb3083aeaaa14008c9240',\n", - " u'question': u'How many Panthers were designated All-Pro?'},\n", - " {u'answers': [{u'answer_start': 85,\n", - " u'text': u'Kelvin Benjamin'},\n", - " {u'answer_start': 85,\n", - " u'text': u'Kelvin Benjamin'},\n", - " {u'answer_start': 92,\n", - " u'text': u'Benjamin'}],\n", - " u'id': u'56beb3083aeaaa14008c9241',\n", - " u'question': u'What Panther tore his ACL in the preseason?'},\n", - " {u'answers': [{u'answer_start': 324,\n", - " u'text': u'1978'},\n", - " {u'answer_start': 324,\n", - " u'text': u'1978'},\n", - " {u'answer_start': 324,\n", - " u'text': u'1978'}],\n", - " u'id': u'56bf2afe3aeaaa14008c9543',\n", - " u'question': u'What year did the league begin having schedules with 16 games in them?'},\n", - " {u'answers': [{u'answer_start': 591,\n", - " u'text': u'2009'},\n", - " {u'answer_start': 591,\n", - " u'text': u'2009'},\n", - " {u'answer_start': 591,\n", - " u'text': u'2009'}],\n", - " u'id': u'56bf2afe3aeaaa14008c9544',\n", - " u'question': u'What year did the the Saints hit a 13-0 record?'},\n", - " {u'answers': [{u'answer_start': 623,\n", - " u'text': u'2011'},\n", - " {u'answer_start': 623,\n", - " u'text': u'2011'},\n", - " {u'answer_start': 623,\n", - " u'text': u'2011'}],\n", - " u'id': u'56bf2afe3aeaaa14008c9545',\n", - " u'question': u'When did the Packers arrive at a record of 13-0?'},\n", - " {u'answers': [{u'answer_start': 106,\n", - " u'text': u'torn ACL'},\n", - " {u'answer_start': 104,\n", - " u'text': u'a torn ACL'},\n", - " {u'answer_start': 106,\n", - " u'text': u'torn ACL'}],\n", - " u'id': u'56bf2afe3aeaaa14008c9547',\n", - " u'question': u'What injury did the Carolina Panthers lose Kelvin Benjamin to during their preseason?'},\n", - " {u'answers': [{u'answer_start': 85,\n", - " u'text': u'Kelvin Benjamin'},\n", - " {u'answer_start': 85,\n", - " u'text': u'Kelvin Benjamin'},\n", - " {u'answer_start': 92,\n", - " u'text': u'Benjamin'}],\n", - " u'id': u'56d6f2000d65d2140019827c',\n", - " u'question': u'Which player did the Panthers lose to an ACL injury in a preseason game?'},\n", - " {u'answers': [{u'answer_start': 38,\n", - " u'text': u'DeAngelo Williams'},\n", - " {u'answer_start': 38,\n", - " u'text': u'DeAngelo Williams'},\n", - " {u'answer_start': 47,\n", - " u'text': u'Williams'}],\n", - " u'id': u'56d6f2000d65d2140019827d',\n", - " u'question': u'Which running back did the Panthers waive?'},\n", - " {u'answers': [{u'answer_start': 324,\n", - " u'text': u'1978'},\n", - " {u'answer_start': 324,\n", - " u'text': u'1978'},\n", - " {u'answer_start': 324,\n", - " u'text': u'1978'}],\n", - " u'id': u'56d6f2000d65d2140019827e',\n", - " u'question': u'When did the NFL start their 16 game seasons?'},\n", - " {u'answers': [{u'answer_start': 807,\n", - " u'text': u'Ten'},\n", - " {u'answer_start': 807,\n", - " u'text': u'Ten'},\n", - " {u'answer_start': 807,\n", - " u'text': u'Ten'}],\n", - " u'id': u'56d6f2000d65d2140019827f',\n", - " u'question': u'How many Panthers players were selected to the Pro Bowl?'},\n", - " {u'answers': [{u'answer_start': 137,\n", - " u'text': u'Carolina Panthers'},\n", - " {u'answer_start': 695,\n", - " u'text': u'the Panthers'},\n", - " {u'answer_start': 330,\n", - " u'text': u'Carolina'}],\n", - " u'id': u'56d9943fdc89441400fdb576',\n", - " u'question': u'Which team had the best regular season in their history?'},\n", - " {u'answers': [{u'answer_start': 324,\n", - " u'text': u'1978.'},\n", - " {u'answer_start': 324,\n", - " u'text': u'1978'},\n", - " {u'answer_start': 324,\n", - " u'text': u'1978'}],\n", - " u'id': u'56d9943fdc89441400fdb577',\n", - " u'question': u'When did the league go from 15 to 16 games in the regular season?'},\n", - " {u'answers': [{u'answer_start': 137,\n", - " u'text': u'Carolina Panthers'},\n", - " {u'answer_start': 695,\n", - " u'text': u'the Panthers'},\n", - " {u'answer_start': 330,\n", - " u'text': u'Carolina'}],\n", - " u'id': u'56d9943fdc89441400fdb578',\n", - " u'question': u'What team had the best start ever in the NFL?'},\n", - " {u'answers': [{u'answer_start': 807,\n", - " u'text': u'Ten'},\n", - " {u'answer_start': 807,\n", - " u'text': u'Ten'},\n", - " {u'answer_start': 807,\n", - " u'text': u'Ten'}],\n", - " u'id': u'56d9943fdc89441400fdb57a',\n", - " u'question': u\"How many Panthers players were chosen for the 2015 season's Pro Bowl?\"}]},\n", - " {u'context': u\"The Panthers offense, which led the NFL in scoring (500 points), was loaded with talent, boasting six Pro Bowl selections. Pro Bowl quarterback Cam Newton had one of his best seasons, throwing for 3,837 yards and rushing for 636, while recording a career-high and league-leading 45 total touchdowns (35 passing, 10 rushing), a career-low 10 interceptions, and a career-best quarterback rating of 99.4. Newton's leading receivers were tight end Greg Olsen, who caught a career-high 77 passes for 1,104 yards and seven touchdowns, and wide receiver Ted Ginn, Jr., who caught 44 passes for 739 yards and 10 touchdowns; Ginn also rushed for 60 yards and returned 27 punts for 277 yards. Other key receivers included veteran Jerricho Cotchery (39 receptions for 485 yards), rookie Devin Funchess (31 receptions for 473 yards and five touchdowns), and second-year receiver Corey Brown (31 receptions for 447 yards). The Panthers backfield featured Pro Bowl running back Jonathan Stewart, who led the team with 989 rushing yards and six touchdowns in 13 games, along with Pro Bowl fullback Mike Tolbert, who rushed for 256 yards and caught 18 passes for another 154 yards. Carolina's offensive line also featured two Pro Bowl selections: center Ryan Kalil and guard Trai Turner.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 98,\n", - " u'text': u'six'},\n", - " {u'answer_start': 98,\n", - " u'text': u'six'},\n", - " {u'answer_start': 98,\n", - " u'text': u'six'}],\n", - " u'id': u'56beb3a03aeaaa14008c9247',\n", - " u'question': u'How many Pro Bowlers were on the Panthers offense?'},\n", - " {u'answers': [{u'answer_start': 279,\n", - " u'text': u'45'},\n", - " {u'answer_start': 279,\n", - " u'text': u'45'},\n", - " {u'answer_start': 279,\n", - " u'text': u'45'}],\n", - " u'id': u'56beb3a03aeaaa14008c9248',\n", - " u'question': u'How many total touchdowns did Cam Newton score?'},\n", - " {u'answers': [{u'answer_start': 312,\n", - " u'text': u'10'},\n", - " {u'answer_start': 338,\n", - " u'text': u'10'},\n", - " {u'answer_start': 338,\n", - " u'text': u'10'}],\n", - " u'id': u'56beb3a03aeaaa14008c9249',\n", - " u'question': u'How many interceptions did Cam Newton throw?'},\n", - " {u'answers': [{u'answer_start': 659,\n", - " u'text': u'27'},\n", - " {u'answer_start': 659,\n", - " u'text': u'27'},\n", - " {u'answer_start': 659,\n", - " u'text': u'27'}],\n", - " u'id': u'56beb3a03aeaaa14008c924a',\n", - " u'question': u'How many punts did Ted Ginn Jr. return?'},\n", - " {u'answers': [{u'answer_start': 444,\n", - " u'text': u'Greg Olsen'},\n", - " {u'answer_start': 444,\n", - " u'text': u'Greg Olsen'},\n", - " {u'answer_start': 449,\n", - " u'text': u'Olsen'}],\n", - " u'id': u'56beb3a03aeaaa14008c924b',\n", - " u'question': u'Who started at tight end for the Panthers?'},\n", - " {u'answers': [{u'answer_start': 279,\n", - " u'text': u'45'},\n", - " {u'answer_start': 279,\n", - " u'text': u'45'},\n", - " {u'answer_start': 279,\n", - " u'text': u'45'}],\n", - " u'id': u'56bf6b303aeaaa14008c960b',\n", - " u'question': u'What is the record number of touchdowns Cam Newton has had? '},\n", - " {u'answers': [{u'answer_start': 396,\n", - " u'text': u'99.4'},\n", - " {u'answer_start': 396,\n", - " u'text': u'99.4'},\n", - " {u'answer_start': 396,\n", - " u'text': u'99.4.'}],\n", - " u'id': u'56bf6b303aeaaa14008c960c',\n", - " u'question': u'What is the best QB ranking that Cam Newton holds?'},\n", - " {u'answers': [{u'answer_start': 481,\n", - " u'text': u'77 passes'},\n", - " {u'answer_start': 481,\n", - " u'text': u'77'},\n", - " {u'answer_start': 481,\n", - " u'text': u'77'}],\n", - " u'id': u'56bf6b303aeaaa14008c960d',\n", - " u'question': u'What are the most passes that Greg Olsen has had?'},\n", - " {u'answers': [{u'answer_start': 419,\n", - " u'text': u'receivers'},\n", - " {u'answer_start': 693,\n", - " u'text': u'receivers'},\n", - " {u'answer_start': 693,\n", - " u'text': u'receivers'}],\n", - " u'id': u'56bf6b303aeaaa14008c960e',\n", - " u'question': u'What position does Jerricho Cotchery play?'},\n", - " {u'answers': [{u'answer_start': 964,\n", - " u'text': u'Jonathan Stewart'},\n", - " {u'answer_start': 964,\n", - " u'text': u'Jonathan Stewart'},\n", - " {u'answer_start': 964,\n", - " u'text': u'Jonathan Stewart'}],\n", - " u'id': u'56bf6b303aeaaa14008c960f',\n", - " u'question': u'Which Panthers RB scored 6 TDs in the 13 games leading up to Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 98,\n", - " u'text': u'six'},\n", - " {u'answer_start': 98,\n", - " u'text': u'six'},\n", - " {u'answer_start': 98,\n", - " u'text': u'six'}],\n", - " u'id': u'56d6f2960d65d21400198286',\n", - " u'question': u'How many Offensive players from the Panthers were selected to play in the Pro Bowl?'},\n", - " {u'answers': [{u'answer_start': 144,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 144,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 144,\n", - " u'text': u'Cam Newton'}],\n", - " u'id': u'56d6f2960d65d21400198287',\n", - " u'question': u'Who is the quarterback for the Carolina Panthers?'},\n", - " {u'answers': [{u'answer_start': 197,\n", - " u'text': u'3,837'},\n", - " {u'answer_start': 197,\n", - " u'text': u'3,837'},\n", - " {u'answer_start': 197,\n", - " u'text': u'3,837'}],\n", - " u'id': u'56d6f2960d65d21400198288',\n", - " u'question': u'How many yards did Newton throw for in 2015?'},\n", - " {u'answers': [{u'answer_start': 279,\n", - " u'text': u'45'},\n", - " {u'answer_start': 279,\n", - " u'text': u'45'},\n", - " {u'answer_start': 279,\n", - " u'text': u'45'}],\n", - " u'id': u'56d6f2960d65d21400198289',\n", - " u'question': u'How many touchdowns did Newton have in 2015?'},\n", - " {u'answers': [{u'answer_start': 98,\n", - " u'text': u'six'},\n", - " {u'answer_start': 1026,\n", - " u'text': u'six'},\n", - " {u'answer_start': 1026,\n", - " u'text': u'six'}],\n", - " u'id': u'56d6f2960d65d2140019828a',\n", - " u'question': u'How many touchdowns did Jonathan Stewart have in 13 games?'},\n", - " {u'answers': [{u'answer_start': 52,\n", - " u'text': u'500'},\n", - " {u'answer_start': 52,\n", - " u'text': u'500'},\n", - " {u'answer_start': 52,\n", - " u'text': u'500'}],\n", - " u'id': u'56d997cddc89441400fdb586',\n", - " u'question': u'How many points did Carolina lead the NFL in scoring for offensive plays?'},\n", - " {u'answers': [{u'answer_start': 197,\n", - " u'text': u'3,837'},\n", - " {u'answer_start': 197,\n", - " u'text': u'3,837'},\n", - " {u'answer_start': 197,\n", - " u'text': u'3,837'}],\n", - " u'id': u'56d997cddc89441400fdb587',\n", - " u'question': u'how many yards did Newton get for passes in the 2015 season?'},\n", - " {u'answers': [{u'answer_start': 279,\n", - " u'text': u'45'},\n", - " {u'answer_start': 279,\n", - " u'text': u'45'},\n", - " {u'answer_start': 279,\n", - " u'text': u'45'}],\n", - " u'id': u'56d997cddc89441400fdb588',\n", - " u'question': u'How many touchdowns did Newton get in the 2015 season?'},\n", - " {u'answers': [{u'answer_start': 396,\n", - " u'text': u'99.4.'},\n", - " {u'answer_start': 396,\n", - " u'text': u'99.4'},\n", - " {u'answer_start': 396,\n", - " u'text': u'99.4.'}],\n", - " u'id': u'56d997cddc89441400fdb589',\n", - " u'question': u\"What was Newton's quarterback rating for 2015?\"},\n", - " {u'answers': [{u'answer_start': 588,\n", - " u'text': u'39'},\n", - " {u'answer_start': 739,\n", - " u'text': u'39'},\n", - " {u'answer_start': 739,\n", - " u'text': u'39'}],\n", - " u'id': u'56d997cddc89441400fdb58a',\n", - " u'question': u'How many receptions did Cotchery get for the 2015 season?'}]},\n", - " {u'context': u\"The Panthers defense gave up just 308 points, ranking sixth in the league, while also leading the NFL in interceptions with 24 and boasting four Pro Bowl selections. Pro Bowl defensive tackle Kawann Short led the team in sacks with 11, while also forcing three fumbles and recovering two. Fellow lineman Mario Addison added 6\\xbd sacks. The Panthers line also featured veteran defensive end Jared Allen, a 5-time pro bowler who was the NFL's active career sack leader with 136, along with defensive end Kony Ealy, who had 5 sacks in just 9 starts. Behind them, two of the Panthers three starting linebackers were also selected to play in the Pro Bowl: Thomas Davis and Luke Kuechly. Davis compiled 5\\xbd sacks, four forced fumbles, and four interceptions, while Kuechly led the team in tackles (118) forced two fumbles, and intercepted four passes of his own. Carolina's secondary featured Pro Bowl safety Kurt Coleman, who led the team with a career high seven interceptions, while also racking up 88 tackles and Pro Bowl cornerback Josh Norman, who developed into a shutdown corner during the season and had four interceptions, two of which were returned for touchdowns.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 34,\n", - " u'text': u'308'},\n", - " {u'answer_start': 34,\n", - " u'text': u'308'},\n", - " {u'answer_start': 34,\n", - " u'text': u'308'}],\n", - " u'id': u'56beb4343aeaaa14008c925b',\n", - " u'question': u'How many points did the Panthers defense surrender?'},\n", - " {u'answers': [{u'answer_start': 470,\n", - " u'text': u'136'},\n", - " {u'answer_start': 470,\n", - " u'text': u'136'},\n", - " {u'answer_start': 470,\n", - " u'text': u'136'}],\n", - " u'id': u'56beb4343aeaaa14008c925c',\n", - " u'question': u'How many career sacks did Jared Allen have?'},\n", - " {u'answers': [{u'answer_start': 789,\n", - " u'text': u'118'},\n", - " {u'answer_start': 789,\n", - " u'text': u'118'},\n", - " {u'answer_start': 789,\n", - " u'text': u'118'}],\n", - " u'id': u'56beb4343aeaaa14008c925d',\n", - " u'question': u'How many tackles did Luke Kuechly register?'},\n", - " {u'answers': [{u'answer_start': 140,\n", - " u'text': u'four'},\n", - " {u'answer_start': 1104,\n", - " u'text': u'four'},\n", - " {u'answer_start': 1104,\n", - " u'text': u'four'}],\n", - " u'id': u'56beb4343aeaaa14008c925e',\n", - " u'question': u'How many balls did Josh Norman intercept?'},\n", - " {u'answers': [{u'answer_start': 192,\n", - " u'text': u'Kawann Short'},\n", - " {u'answer_start': 192,\n", - " u'text': u'Kawann Short'},\n", - " {u'answer_start': 199,\n", - " u'text': u'Short'}],\n", - " u'id': u'56beb4343aeaaa14008c925f',\n", - " u'question': u'Who registered the most sacks on the team this season?'},\n", - " {u'answers': [{u'answer_start': 124,\n", - " u'text': u'24'},\n", - " {u'answer_start': 124,\n", - " u'text': u'24'},\n", - " {u'answer_start': 124,\n", - " u'text': u'24'}],\n", - " u'id': u'56d6f3500d65d21400198290',\n", - " u'question': u'How many interceptions are the Panthers defense credited with in 2015?'},\n", - " {u'answers': [{u'answer_start': 192,\n", - " u'text': u'Kawann Short'},\n", - " {u'answer_start': 192,\n", - " u'text': u'Kawann Short'},\n", - " {u'answer_start': 199,\n", - " u'text': u'Short'}],\n", - " u'id': u'56d6f3500d65d21400198291',\n", - " u'question': u'Who led the Panthers in sacks?'},\n", - " {u'answers': [{u'answer_start': 140,\n", - " u'text': u'four'},\n", - " {u'answer_start': 140,\n", - " u'text': u'four'},\n", - " {u'answer_start': 140,\n", - " u'text': u'four'}],\n", - " u'id': u'56d6f3500d65d21400198292',\n", - " u'question': u'How many Panthers defense players were selected for the Pro Bowl?'},\n", - " {u'answers': [{u'answer_start': 140,\n", - " u'text': u'four'},\n", - " {u'answer_start': 695,\n", - " u'text': u'5\\xbd'},\n", - " {u'answer_start': 705,\n", - " u'text': u'four'}],\n", - " u'id': u'56d6f3500d65d21400198293',\n", - " u'question': u'How many forced fumbles did Thomas Davis have?'},\n", - " {u'answers': [{u'answer_start': 900,\n", - " u'text': u'Kurt Coleman'},\n", - " {u'answer_start': 900,\n", - " u'text': u'Kurt Coleman'},\n", - " {u'answer_start': 905,\n", - " u'text': u'Coleman'}],\n", - " u'id': u'56d6f3500d65d21400198294',\n", - " u'question': u'Which player had the most interceptions for the season?'},\n", - " {u'answers': [{u'answer_start': 124,\n", - " u'text': u'24'},\n", - " {u'answer_start': 124,\n", - " u'text': u'24'},\n", - " {u'answer_start': 124,\n", - " u'text': u'24'}],\n", - " u'id': u'56d9992fdc89441400fdb59c',\n", - " u'question': u\"How many 2015 season interceptions did the Panthers' defense get?\"},\n", - " {u'answers': [{u'answer_start': 500,\n", - " u'text': u'Kony Ealy'},\n", - " {u'answer_start': 500,\n", - " u'text': u'Kony Ealy'}],\n", - " u'id': u'56d9992fdc89441400fdb59e',\n", - " u'question': u'Who had five sacks in nine games as a Carolina Panthers starter?'},\n", - " {u'answers': [{u'answer_start': 666,\n", - " u'text': u'Luke Kuechly.'},\n", - " {u'answer_start': 666,\n", - " u'text': u'Luke Kuechly'},\n", - " {u'answer_start': 199,\n", - " u'text': u'Short'}],\n", - " u'id': u'56d9992fdc89441400fdb59f',\n", - " u'question': u\"Who was the Panthers' tackle leader for 2015?\"},\n", - " {u'answers': [{u'answer_start': 284,\n", - " u'text': u'two.'},\n", - " {u'answer_start': 1124,\n", - " u'text': u'two'},\n", - " {u'answer_start': 124,\n", - " u'text': u'24'}],\n", - " u'id': u'56d9992fdc89441400fdb5a0',\n", - " u'question': u'How many interceptions did Josh Norman score touchdowns with in 2015?'}]},\n", - " {u'context': u\"Following their loss in the divisional round of the previous season's playoffs, the Denver Broncos underwent numerous coaching changes, including a mutual parting with head coach John Fox (who had won four divisional championships in his four years as Broncos head coach), and the hiring of Gary Kubiak as the new head coach. Under Kubiak, the Broncos planned to install a run-oriented offense with zone blocking to blend in with quarterback Peyton Manning's shotgun passing skills, but struggled with numerous changes and injuries to the offensive line, as well as Manning having his worst statistical season since his rookie year with the Indianapolis Colts in 1998, due to a plantar fasciitis injury in his heel that he had suffered since the summer, and the simple fact that Manning was getting old, as he turned 39 in the 2015 off-season. Although the team had a 7\\u20130 start, Manning led the NFL in interceptions. In week 10, Manning suffered a partial tear of the plantar fasciitis in his left foot. He set the NFL's all-time record for career passing yards in this game, but was benched after throwing four interceptions in favor of backup quarterback Brock Osweiler, who took over as the starter for most of the remainder of the regular season. Osweiler was injured, however, leading to Manning's return during the Week 17 regular season finale, where the Broncos were losing 13\\u20137 against the 4\\u201311 San Diego Chargers, resulting in Manning re-claiming the starting quarterback position for the playoffs by leading the team to a key 27\\u201320 win that enabled the team to clinch the number one overall AFC seed. Under defensive coordinator Wade Phillips, the Broncos' defense ranked number one in total yards allowed, passing yards allowed and sacks, and like the previous three seasons, the team has continued to set numerous individual, league and franchise records. With the defense carrying the team despite the issues with the offense, the Broncos finished the regular season with a 12\\u20134 record and earned home-field advantage throughout the AFC playoffs.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 291,\n", - " u'text': u'Gary Kubiak'},\n", - " {u'answer_start': 291,\n", - " u'text': u'Gary Kubiak'},\n", - " {u'answer_start': 332,\n", - " u'text': u'Kubiak'}],\n", - " u'id': u'56beb4e43aeaaa14008c9265',\n", - " u'question': u'Who is the head coach of the Broncos?'},\n", - " {u'answers': [{u'answer_start': 1157,\n", - " u'text': u'Brock Osweiler'},\n", - " {u'answer_start': 1157,\n", - " u'text': u'Brock Osweiler'},\n", - " {u'answer_start': 1163,\n", - " u'text': u'Osweiler'}],\n", - " u'id': u'56beb4e43aeaaa14008c9266',\n", - " u'question': u'Who played quarterback for the Broncos after Peyton Manning was benched?'},\n", - " {u'answers': [{u'answer_start': 641,\n", - " u'text': u'Indianapolis Colts'},\n", - " {u'answer_start': 637,\n", - " u'text': u'the Indianapolis Colts'},\n", - " {u'answer_start': 654,\n", - " u'text': u'Colts'}],\n", - " u'id': u'56beb4e43aeaaa14008c9267',\n", - " u'question': u'Who did Peyton Manning play for as a rookie?'},\n", - " {u'answers': [{u'answer_start': 1404,\n", - " u'text': u'San Diego Chargers'},\n", - " {u'answer_start': 1404,\n", - " u'text': u'San Diego Chargers'},\n", - " {u'answer_start': 1414,\n", - " u'text': u'Chargers'}],\n", - " u'id': u'56beb4e43aeaaa14008c9268',\n", - " u'question': u'Who did the Broncos play in the last week of the regular season?'},\n", - " {u'answers': [{u'answer_start': 1640,\n", - " u'text': u'Wade Phillips'},\n", - " {u'answer_start': 1640,\n", - " u'text': u'Wade Phillips'},\n", - " {u'answer_start': 1645,\n", - " u'text': u'Phillips'}],\n", - " u'id': u'56beb4e43aeaaa14008c9269',\n", - " u'question': u\"Who is Denver's defensive coordinator?\"},\n", - " {u'answers': [{u'answer_start': 201,\n", - " u'text': u'four'},\n", - " {u'answer_start': 238,\n", - " u'text': u'four'},\n", - " {u'answer_start': 238,\n", - " u'text': u'four'}],\n", - " u'id': u'56bf301c3aeaaa14008c954d',\n", - " u'question': u'How many years was John Fox the head coach of the Denver Broncos?'},\n", - " {u'answers': [{u'answer_start': 291,\n", - " u'text': u'Gary Kubiak'},\n", - " {u'answer_start': 291,\n", - " u'text': u'Gary Kubiak'},\n", - " {u'answer_start': 332,\n", - " u'text': u'Kubiak'}],\n", - " u'id': u'56bf301c3aeaaa14008c954e',\n", - " u'question': u\"What is the name of the Bronco's head coach, who was hired after John Fox? \"},\n", - " {u'answers': [{u'answer_start': 641,\n", - " u'text': u'Indianapolis Colts'},\n", - " {u'answer_start': 637,\n", - " u'text': u'the Indianapolis Colts'},\n", - " {u'answer_start': 663,\n", - " u'text': u'1998'}],\n", - " u'id': u'56bf301c3aeaaa14008c954f',\n", - " u'question': u'What was the first team Peyton Manning began playing for?'},\n", - " {u'answers': [{u'answer_start': 817,\n", - " u'text': u'39'},\n", - " {u'answer_start': 817,\n", - " u'text': u'39'},\n", - " {u'answer_start': 817,\n", - " u'text': u'39'}],\n", - " u'id': u'56bf301c3aeaaa14008c9550',\n", - " u'question': u'How old was Peyton Manning in 2015?'},\n", - " {u'answers': [{u'answer_start': 678,\n", - " u'text': u'plantar fasciitis'},\n", - " {u'answer_start': 946,\n", - " u'text': u'a partial tear of the plantar fasciitis'}],\n", - " u'id': u'56bf301c3aeaaa14008c9551',\n", - " u'question': u'In the 10th week of the 2015 season, what injury was Peyton Manning dealing with?'},\n", - " {u'answers': [{u'answer_start': 291,\n", - " u'text': u'Gary Kubiak'},\n", - " {u'answer_start': 291,\n", - " u'text': u'Gary Kubiak'},\n", - " {u'answer_start': 332,\n", - " u'text': u'Kubiak'}],\n", - " u'id': u'56d6f4030d65d2140019829a',\n", - " u'question': u'When John Fox left as head coach for the Broncos, who replaced him?'},\n", - " {u'answers': [{u'answer_start': 442,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 442,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 566,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56d6f4030d65d2140019829b',\n", - " u'question': u'Who was the quarterback for the Broncos 2015 season?'},\n", - " {u'answers': [{u'answer_start': 676,\n", - " u'text': u'a plantar fasciitis injury'},\n", - " {u'answer_start': 676,\n", - " u'text': u'a plantar fasciitis injury'},\n", - " {u'answer_start': 678,\n", - " u'text': u'plantar fasciitis'}],\n", - " u'id': u'56d6f4030d65d2140019829c',\n", - " u'question': u'What injury did Manning suffer the summer before the season started?'},\n", - " {u'answers': [{u'answer_start': 817,\n", - " u'text': u'39'},\n", - " {u'answer_start': 817,\n", - " u'text': u'39'},\n", - " {u'answer_start': 817,\n", - " u'text': u'39'}],\n", - " u'id': u'56d6f4030d65d2140019829d',\n", - " u'question': u'How old was Manning at the beginning of the 2015 season?'},\n", - " {u'answers': [{u'answer_start': 201,\n", - " u'text': u'four'},\n", - " {u'answer_start': 1991,\n", - " u'text': u'4'},\n", - " {u'answer_start': 1991,\n", - " u'text': u'4'}],\n", - " u'id': u'56d6f4030d65d2140019829e',\n", - " u'question': u'How many games did the Broncos lose during their regular 2015 season?'},\n", - " {u'answers': [{u'answer_start': 179,\n", - " u'text': u'John Fox'},\n", - " {u'answer_start': 179,\n", - " u'text': u'John Fox'},\n", - " {u'answer_start': 184,\n", - " u'text': u'Fox'}],\n", - " u'id': u'56d99b7bdc89441400fdb5c8',\n", - " u'question': u'What coach left the Broncos after the season prior to Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 442,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 442,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 566,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56d99b7bdc89441400fdb5c9',\n", - " u'question': u'According to game stats, which Super Bowl 50 quarterback had his worst year since his first season as a player in the NFL?'},\n", - " {u'answers': [{u'answer_start': 291,\n", - " u'text': u'Gary Kubiak'},\n", - " {u'answer_start': 291,\n", - " u'text': u'Gary Kubiak'},\n", - " {u'answer_start': 296,\n", - " u'text': u'Kubiak'}],\n", - " u'id': u'56d99b7bdc89441400fdb5ca',\n", - " u'question': u'Who was the Denver head coach for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 993,\n", - " u'text': u'left foot.'},\n", - " {u'answer_start': 993,\n", - " u'text': u'left'},\n", - " {u'answer_start': 993,\n", - " u'text': u'left'}],\n", - " u'id': u'56d99b7bdc89441400fdb5cb',\n", - " u'question': u'What foot was injured on Manning that sidelined him in week 10?'},\n", - " {u'answers': [{u'answer_start': 1640,\n", - " u'text': u'Wade Phillips'},\n", - " {u'answer_start': 1640,\n", - " u'text': u'Wade Phillips'},\n", - " {u'answer_start': 1645,\n", - " u'text': u'Phillips'}],\n", - " u'id': u'56d99b7bdc89441400fdb5cc',\n", - " u'question': u'Who was the defensive coordinator for the Broncos in 2015?'}]},\n", - " {u'context': u\"Manning finished the year with a career-low 67.9 passer rating, throwing for 2,249 yards and nine touchdowns, with 17 interceptions. In contrast, Osweiler threw for 1,967 yards, 10 touchdowns and six interceptions for a rating of 86.4. Veteran receiver Demaryius Thomas led the team with 105 receptions for 1,304 yards and six touchdowns, while Emmanuel Sanders caught 76 passes for 1,135 yards and six scores, while adding another 106 yards returning punts. Tight end Owen Daniels was also a big element of the passing game with 46 receptions for 517 yards. Running back C. J. Anderson was the team's leading rusher 863 yards and seven touchdowns, while also catching 25 passes for 183 yards. Running back Ronnie Hillman also made a big impact with 720 yards, five touchdowns, 24 receptions, and a 4.7 yards per carry average. Overall, the offense ranked 19th in scoring with 355 points and did not have any Pro Bowl selections.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 44,\n", - " u'text': u'67.9'},\n", - " {u'answer_start': 44,\n", - " u'text': u'67.9'},\n", - " {u'answer_start': 44,\n", - " u'text': u'67.9'}],\n", - " u'id': u'56beb57b3aeaaa14008c9279',\n", - " u'question': u\"What was Peyton Manning's passer rating for the season?\"},\n", - " {u'answers': [{u'answer_start': 115,\n", - " u'text': u'17'},\n", - " {u'answer_start': 115,\n", - " u'text': u'17'},\n", - " {u'answer_start': 115,\n", - " u'text': u'17'}],\n", - " u'id': u'56beb57b3aeaaa14008c927a',\n", - " u'question': u'How many picks did Peyton Manning throw in the season?'},\n", - " {u'answers': [{u'answer_start': 253,\n", - " u'text': u'Demaryius Thomas'},\n", - " {u'answer_start': 253,\n", - " u'text': u'Demaryius Thomas'},\n", - " {u'answer_start': 263,\n", - " u'text': u'Thomas'}],\n", - " u'id': u'56beb57b3aeaaa14008c927b',\n", - " u'question': u'Who registered the most receptions on the Broncos?'},\n", - " {u'answers': [{u'answer_start': 572,\n", - " u'text': u'C. J. Anderson'},\n", - " {u'answer_start': 572,\n", - " u'text': u'C. J. Anderson'},\n", - " {u'answer_start': 578,\n", - " u'text': u'Anderson'}],\n", - " u'id': u'56beb57b3aeaaa14008c927c',\n", - " u'question': u'Who had the most rushing yards on the Broncos?'},\n", - " {u'answers': [{u'answer_start': 178,\n", - " u'text': u'10'},\n", - " {u'answer_start': 178,\n", - " u'text': u'10'},\n", - " {u'answer_start': 178,\n", - " u'text': u'10'}],\n", - " u'id': u'56beb57b3aeaaa14008c927d',\n", - " u'question': u'How many touchdowns did Brock Osweiler throw in the season?'},\n", - " {u'answers': [{u'answer_start': 44,\n", - " u'text': u'67.9'},\n", - " {u'answer_start': 44,\n", - " u'text': u'67.9'},\n", - " {u'answer_start': 44,\n", - " u'text': u'67.9'}],\n", - " u'id': u'56bf38383aeaaa14008c956b',\n", - " u'question': u'What was the passer rating for Peyton Manning that he finished out with for the season? '},\n", - " {u'answers': [{u'answer_start': 77,\n", - " u'text': u'2,249'},\n", - " {u'answer_start': 77,\n", - " u'text': u'2,249'},\n", - " {u'answer_start': 77,\n", - " u'text': u'2,249'}],\n", - " u'id': u'56bf38383aeaaa14008c956c',\n", - " u'question': u'How may yards did Peyton Manning throw?'},\n", - " {u'answers': [{u'answer_start': 93,\n", - " u'text': u'nine'},\n", - " {u'answer_start': 115,\n", - " u'text': u'17'},\n", - " {u'answer_start': 33,\n", - " u'text': u'career-low 67.9 passer rating,'}],\n", - " u'id': u'56bf38383aeaaa14008c956d',\n", - " u'question': u'What was the number of 17 interceptions that Peyton Manning had by end of the year?'},\n", - " {u'answers': [{u'answer_start': 253,\n", - " u'text': u'Demaryius Thomas'},\n", - " {u'answer_start': 253,\n", - " u'text': u'Demaryius Thomas'},\n", - " {u'answer_start': 263,\n", - " u'text': u'Thomas'}],\n", - " u'id': u'56bf38383aeaaa14008c956e',\n", - " u'question': u'Who had the most receptions out of all players for the year?'},\n", - " {u'answers': [{u'answer_start': 244,\n", - " u'text': u'receiver'},\n", - " {u'answer_start': 244,\n", - " u'text': u'receiver'},\n", - " {u'answer_start': 263,\n", - " u'text': u'Thomas'}],\n", - " u'id': u'56bf38383aeaaa14008c956f',\n", - " u'question': u'What position does Demaryius Thomas play?'},\n", - " {u'answers': [{u'answer_start': 44,\n", - " u'text': u'67.9'},\n", - " {u'answer_start': 44,\n", - " u'text': u'67.9'},\n", - " {u'answer_start': 44,\n", - " u'text': u'67.9'}],\n", - " u'id': u'56d6fe0b0d65d214001982a4',\n", - " u'question': u\"What was Manning's passer rating at the end of the season?\"},\n", - " {u'answers': [{u'answer_start': 115,\n", - " u'text': u'17'},\n", - " {u'answer_start': 115,\n", - " u'text': u'17'},\n", - " {u'answer_start': 115,\n", - " u'text': u'17'}],\n", - " u'id': u'56d6fe0b0d65d214001982a5',\n", - " u'question': u'How many times was Manning intercepted during the 2015 season?'},\n", - " {u'answers': [{u'answer_start': 253,\n", - " u'text': u'Demaryius Thomas'},\n", - " {u'answer_start': 253,\n", - " u'text': u'Demaryius Thomas'},\n", - " {u'answer_start': 263,\n", - " u'text': u'Thomas'}],\n", - " u'id': u'56d6fe0b0d65d214001982a6',\n", - " u'question': u'Who led the Broncos with 105 receptions?'},\n", - " {u'answers': [{u'answer_start': 290,\n", - " u'text': u'5'},\n", - " {u'answer_start': 761,\n", - " u'text': u'five'},\n", - " {u'answer_start': 761,\n", - " u'text': u'five'}],\n", - " u'id': u'56d6fe0b0d65d214001982a7',\n", - " u'question': u'How many touchdowns did Ronnie Hillman make?'},\n", - " {u'answers': [{u'answer_start': 44,\n", - " u'text': u'67.9'},\n", - " {u'answer_start': 44,\n", - " u'text': u'67.9'},\n", - " {u'answer_start': 44,\n", - " u'text': u'67.9'}],\n", - " u'id': u'56d99c44dc89441400fdb5d6',\n", - " u'question': u\"What was Manning's passer rating for the 2015 season?\"},\n", - " {u'answers': [{u'answer_start': 115,\n", - " u'text': u'17'},\n", - " {u'answer_start': 115,\n", - " u'text': u'17'},\n", - " {u'answer_start': 115,\n", - " u'text': u'17'}],\n", - " u'id': u'56d99c44dc89441400fdb5d7',\n", - " u'question': u'How many interceptions did manning have in 2015?'},\n", - " {u'answers': [{u'answer_start': 345,\n", - " u'text': u'Emmanuel Sanders'},\n", - " {u'answer_start': 345,\n", - " u'text': u'Emmanuel Sanders'},\n", - " {u'answer_start': 354,\n", - " u'text': u'Sanders'}],\n", - " u'id': u'56d99c44dc89441400fdb5d8',\n", - " u'question': u'What Denver player caught the ball 76 times in the 2015 season?'},\n", - " {u'answers': [{u'answer_start': 572,\n", - " u'text': u'C. J. Anderson'},\n", - " {u'answer_start': 572,\n", - " u'text': u'C. J. Anderson'},\n", - " {u'answer_start': 578,\n", - " u'text': u'Anderson'}],\n", - " u'id': u'56d99c44dc89441400fdb5d9',\n", - " u'question': u\"Who was the Broncos' leading rusher for the 2015 season?\"},\n", - " {u'answers': [{u'answer_start': 799,\n", - " u'text': u'4.7'},\n", - " {u'answer_start': 799,\n", - " u'text': u'4.7'},\n", - " {u'answer_start': 799,\n", - " u'text': u'4.7'}],\n", - " u'id': u'56d99c44dc89441400fdb5da',\n", - " u'question': u\"What was Ronnie Hillman's average yards per carry in 2015?\"}]},\n", - " {u'context': u\"The Broncos' defense ranked first in the NFL yards allowed (4,530) for the first time in franchise history, and fourth in points allowed (296). Defensive ends Derek Wolfe and Malik Jackson each had 5\\xbd sacks. Pro Bowl linebacker Von Miller led the team with 11 sacks, forced four fumbles, and recovered three. Linebacker DeMarcus Ware was selected to play in the Pro Bowl for the ninth time in his career, ranking second on the team with 7\\xbd sacks. Linebacker Brandon Marshall led the team in total tackles with 109, while Danny Trevathan ranked second with 102. Cornerbacks Aqib Talib (three interceptions) and Chris Harris, Jr. (two interceptions) were the other two Pro Bowl selections from the defense.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 60,\n", - " u'text': u'4,530'},\n", - " {u'answer_start': 60,\n", - " u'text': u'4,530'},\n", - " {u'answer_start': 60,\n", - " u'text': u'4,530'}],\n", - " u'id': u'56beb6533aeaaa14008c928d',\n", - " u'question': u\"How many yards did the Broncos' defense give up?\"},\n", - " {u'answers': [{u'answer_start': 198,\n", - " u'text': u'5\\xbd'},\n", - " {u'answer_start': 198,\n", - " u'text': u'5\\xbd'},\n", - " {u'answer_start': 198,\n", - " u'text': u'5\\xbd'}],\n", - " u'id': u'56beb6533aeaaa14008c928e',\n", - " u'question': u'How many sacks did Derek Wolfe register?'},\n", - " {u'answers': [{u'answer_start': 458,\n", - " u'text': u'Brandon Marshall'},\n", - " {u'answer_start': 458,\n", - " u'text': u'Brandon Marshall'},\n", - " {u'answer_start': 466,\n", - " u'text': u'Marshall'}],\n", - " u'id': u'56beb6533aeaaa14008c9290',\n", - " u'question': u'Who was first on the team in total tackles?'},\n", - " {u'answers': [{u'answer_start': 302,\n", - " u'text': u'three'},\n", - " {u'answer_start': 585,\n", - " u'text': u'three'},\n", - " {u'answer_start': 585,\n", - " u'text': u'three'}],\n", - " u'id': u'56beb6533aeaaa14008c9291',\n", - " u'question': u'How many picks did Aqib Talib have?'},\n", - " {u'answers': [{u'answer_start': 309,\n", - " u'text': u'Linebacker'},\n", - " {u'answer_start': 309,\n", - " u'text': u'Linebacker'},\n", - " {u'answer_start': 309,\n", - " u'text': u'Linebacker'}],\n", - " u'id': u'56bf3e803aeaaa14008c9588',\n", - " u'question': u'What position does DeMarcus currently play? '},\n", - " {u'answers': [{u'answer_start': 309,\n", - " u'text': u'Linebacker'},\n", - " {u'answer_start': 447,\n", - " u'text': u'Linebacker'},\n", - " {u'answer_start': 447,\n", - " u'text': u'Linebacker'}],\n", - " u'id': u'56bf3e803aeaaa14008c9589',\n", - " u'question': u'What position does Brandon Marshall currently play? '},\n", - " {u'answers': [{u'answer_start': 144,\n", - " u'text': u'Defensive ends'},\n", - " {u'answer_start': 144,\n", - " u'text': u'Defensive ends'},\n", - " {u'answer_start': 144,\n", - " u'text': u'Defensive ends'}],\n", - " u'id': u'56bf3e803aeaaa14008c958b',\n", - " u'question': u'What is the position Derek Wolfe plays currently? '},\n", - " {u'answers': [{u'answer_start': 138,\n", - " u'text': u'296'},\n", - " {u'answer_start': 138,\n", - " u'text': u'296'},\n", - " {u'answer_start': 138,\n", - " u'text': u'296'}],\n", - " u'id': u'56d6fea90d65d214001982ae',\n", - " u'question': u'How many points did the Broncos defense allow their opponents to get?'},\n", - " {u'answers': [{u'answer_start': 228,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 228,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 232,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56d6fea90d65d214001982af',\n", - " u'question': u'Which linebacker had the most sacks on the team?'},\n", - " {u'answers': [{u'answer_start': 458,\n", - " u'text': u'Brandon Marshall'},\n", - " {u'answer_start': 458,\n", - " u'text': u'Brandon Marshall'},\n", - " {u'answer_start': 466,\n", - " u'text': u'Marshall'}],\n", - " u'id': u'56d6fea90d65d214001982b0',\n", - " u'question': u'Which linebacker led the Broncos in tackles?'},\n", - " {u'answers': [{u'answer_start': 302,\n", - " u'text': u'three.'},\n", - " {u'answer_start': 585,\n", - " u'text': u'three'},\n", - " {u'answer_start': 585,\n", - " u'text': u'three'}],\n", - " u'id': u'56d6fea90d65d214001982b2',\n", - " u'question': u'How many interceptions did Aqib Talib have for the regular season?'},\n", - " {u'answers': [{u'answer_start': 228,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 228,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 232,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56d99da8dc89441400fdb5fd',\n", - " u'question': u'Who forced four fumbles for the Broncos in the 2015 season?'},\n", - " {u'answers': [{u'answer_start': 447,\n", - " u'text': u'Linebacker Brandon Marshall'},\n", - " {u'answer_start': 458,\n", - " u'text': u'Brandon Marshall'},\n", - " {u'answer_start': 466,\n", - " u'text': u'Marshall'}],\n", - " u'id': u'56d99da8dc89441400fdb5ff',\n", - " u'question': u'What Denver player had 109 tackels for the 2015 season?'},\n", - " {u'answers': [{u'answer_start': 159,\n", - " u'text': u'Derek Wolfe and Malik Jackson'},\n", - " {u'answer_start': 159,\n", - " u'text': u'Derek Wolfe and Malik Jackson'}],\n", - " u'id': u'56d99da8dc89441400fdb600',\n", - " u'question': u'What two Denver players ranked at 5 percent for sacks?'}]},\n", - " {u'context': u'The Panthers beat the Seattle Seahawks in the divisional round, running up a 31\\u20130 halftime lead and then holding off a furious second half comeback attempt to win 31\\u201324, avenging their elimination from a year earlier. The Panthers then blew out the Arizona Cardinals in the NFC Championship Game, 49\\u201315, racking up 487 yards and forcing seven turnovers.',\n", - " u'qas': [{u'answers': [{u'answer_start': 22,\n", - " u'text': u'Seattle Seahawks'},\n", - " {u'answer_start': 18,\n", - " u'text': u'the Seattle Seahawks'},\n", - " {u'answer_start': 30,\n", - " u'text': u'Seahawks'}],\n", - " u'id': u'56beb6f23aeaaa14008c92a1',\n", - " u'question': u'Who lost to the Panthers in the divisional round of the playoffs?'},\n", - " {u'answers': [{u'answer_start': 249,\n", - " u'text': u'Arizona Cardinals'},\n", - " {u'answer_start': 249,\n", - " u'text': u'Arizona Cardinals'},\n", - " {u'answer_start': 257,\n", - " u'text': u'Cardinals'}],\n", - " u'id': u'56beb6f23aeaaa14008c92a2',\n", - " u'question': u'Who did the Panthers face in the NFC Championship Game?'},\n", - " {u'answers': [{u'answer_start': 315,\n", - " u'text': u'487'},\n", - " {u'answer_start': 315,\n", - " u'text': u'487'},\n", - " {u'answer_start': 315,\n", - " u'text': u'487'}],\n", - " u'id': u'56beb6f23aeaaa14008c92a3',\n", - " u'question': u'How many total yards did the Panthers have in the NFC Championship?'},\n", - " {u'answers': [{u'answer_start': 337,\n", - " u'text': u'seven'},\n", - " {u'answer_start': 337,\n", - " u'text': u'seven'},\n", - " {u'answer_start': 337,\n", - " u'text': u'seven'}],\n", - " u'id': u'56beb6f23aeaaa14008c92a4',\n", - " u'question': u'How many times did Arizona turn the ball over in the NFC Championship?'},\n", - " {u'answers': [{u'answer_start': 163,\n", - " u'text': u'31\\u201324'},\n", - " {u'answer_start': 163,\n", - " u'text': u'31\\u201324'},\n", - " {u'answer_start': 163,\n", - " u'text': u'31\\u201324'}],\n", - " u'id': u'56beb6f23aeaaa14008c92a5',\n", - " u'question': u'What was the final score of the game between the Panthers and the Seahawks?'},\n", - " {u'answers': [{u'answer_start': 22,\n", - " u'text': u'Seattle Seahawks'},\n", - " {u'answer_start': 18,\n", - " u'text': u'the Seattle Seahawks'},\n", - " {u'answer_start': 30,\n", - " u'text': u'Seahawks'}],\n", - " u'id': u'56bf34293aeaaa14008c9559',\n", - " u'question': u'What was the team the Carolina Panthers played immediately prior to the NFC Championship game? '},\n", - " {u'answers': [{u'answer_start': 163,\n", - " u'text': u'31\\u201324'},\n", - " {u'answer_start': 163,\n", - " u'text': u'31\\u201324'},\n", - " {u'answer_start': 163,\n", - " u'text': u'31\\u201324,'}],\n", - " u'id': u'56bf34293aeaaa14008c955a',\n", - " u'question': u'What was the score of the last game the Carolina Panthers played prior to the NFC Championship?'},\n", - " {u'answers': [{u'answer_start': 315,\n", - " u'text': u'487'},\n", - " {u'answer_start': 315,\n", - " u'text': u'487'},\n", - " {u'answer_start': 315,\n", - " u'text': u'487'}],\n", - " u'id': u'56bf34293aeaaa14008c955b',\n", - " u'question': u'How many yards did the Carolina Panthers gain in the 487 in the NFC Championship Game?'},\n", - " {u'answers': [{u'answer_start': 22,\n", - " u'text': u'Seattle Seahawks'},\n", - " {u'answer_start': 18,\n", - " u'text': u'the Seattle Seahawks'},\n", - " {u'answer_start': 30,\n", - " u'text': u'Seahawks'}],\n", - " u'id': u'56d701280d65d214001982b8',\n", - " u'question': u'Who did Carolina beat in the divisional round?'},\n", - " {u'answers': [{u'answer_start': 249,\n", - " u'text': u'Arizona Cardinals'},\n", - " {u'answer_start': 245,\n", - " u'text': u'the Arizona Cardinals'},\n", - " {u'answer_start': 257,\n", - " u'text': u'Cardinals'}],\n", - " u'id': u'56d701280d65d214001982ba',\n", - " u'question': u'Who did the Panthers play to advance to the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 337,\n", - " u'text': u'seven'},\n", - " {u'answer_start': 337,\n", - " u'text': u'seven'},\n", - " {u'answer_start': 337,\n", - " u'text': u'seven'}],\n", - " u'id': u'56d701280d65d214001982bc',\n", - " u'question': u'How many turnovers did the Panthers force during the NFC Championship game?'},\n", - " {u'answers': [{u'answer_start': 22,\n", - " u'text': u'Seattle Seahawks'},\n", - " {u'answer_start': 18,\n", - " u'text': u'the Seattle Seahawks'},\n", - " {u'answer_start': 30,\n", - " u'text': u'Seahawks'}],\n", - " u'id': u'56d99e78dc89441400fdb616',\n", - " u'question': u'Who did the Panthers beat in the divisional round?'},\n", - " {u'answers': [{u'answer_start': 297,\n", - " u'text': u'49\\u201315'},\n", - " {u'answer_start': 297,\n", - " u'text': u'49\\u201315'},\n", - " {u'answer_start': 297,\n", - " u'text': u'49\\u201315,'}],\n", - " u'id': u'56d99e78dc89441400fdb618',\n", - " u'question': u'What was the score for the 2015 NFC championship game?'},\n", - " {u'answers': [{u'answer_start': 249,\n", - " u'text': u'Arizona Cardinals'},\n", - " {u'answer_start': 18,\n", - " u'text': u'the Seattle Seahawks'},\n", - " {u'answer_start': 257,\n", - " u'text': u'Cardinals'}],\n", - " u'id': u'56d99e78dc89441400fdb619',\n", - " u'question': u'Who did the Panthers play in their division championship game?'},\n", - " {u'answers': [{u'answer_start': 315,\n", - " u'text': u'487'},\n", - " {u'answer_start': 315,\n", - " u'text': u'487'}],\n", - " u'id': u'56d99e78dc89441400fdb61a',\n", - " u'question': u'How many yards did the Panthers get for the division championshipt game?'}]},\n", - " {u'context': u\"The Broncos defeated the Pittsburgh Steelers in the divisional round, 23\\u201316, by scoring 11 points in the final three minutes of the game. They then beat the defending Super Bowl XLIX champion New England Patriots in the AFC Championship Game, 20\\u201318, by intercepting a pass on New England's 2-point conversion attempt with 17 seconds left on the clock. Despite Manning's problems with interceptions during the season, he didn't throw any in their two playoff games.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 25,\n", - " u'text': u'Pittsburgh Steelers'},\n", - " {u'answer_start': 21,\n", - " u'text': u'the Pittsburgh Steelers'},\n", - " {u'answer_start': 36,\n", - " u'text': u'Steelers'}],\n", - " u'id': u'56beb7953aeaaa14008c92ab',\n", - " u'question': u'Who lost to the Broncos in the divisional round?'},\n", - " {u'answers': [{u'answer_start': 88,\n", - " u'text': u'11'},\n", - " {u'answer_start': 88,\n", - " u'text': u'11'},\n", - " {u'answer_start': 88,\n", - " u'text': u'11'}],\n", - " u'id': u'56beb7953aeaaa14008c92ac',\n", - " u'question': u'How many points did the Broncos score in the last three minutes of the game versus Pittsburgh?'},\n", - " {u'answers': [{u'answer_start': 192,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 192,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 204,\n", - " u'text': u'Patriots'}],\n", - " u'id': u'56beb7953aeaaa14008c92ad',\n", - " u'question': u'Who won Super Bowl XLIX?'},\n", - " {u'answers': [{u'answer_start': 243,\n", - " u'text': u'20\\u201318'},\n", - " {u'answer_start': 243,\n", - " u'text': u'20\\u201318'},\n", - " {u'answer_start': 243,\n", - " u'text': u'20\\u201318,'}],\n", - " u'id': u'56beb7953aeaaa14008c92ae',\n", - " u'question': u'What was the final score of the AFC Championship Game?'},\n", - " {u'answers': [{u'answer_start': 322,\n", - " u'text': u'17 seconds'},\n", - " {u'answer_start': 322,\n", - " u'text': u'17 seconds'},\n", - " {u'answer_start': 322,\n", - " u'text': u'17'}],\n", - " u'id': u'56beb7953aeaaa14008c92af',\n", - " u'question': u'How much time remained on the clock when the Broncos made the interception that clinched the AFC Championship Game?'},\n", - " {u'answers': [{u'answer_start': 4,\n", - " u'text': u'Broncos'},\n", - " {u'answer_start': 0,\n", - " u'text': u'The Broncos'},\n", - " {u'answer_start': 4,\n", - " u'text': u'Broncos'}],\n", - " u'id': u'56bf36b93aeaaa14008c9561',\n", - " u'question': u'What team was the divisional round winner between the Broncos and Steelers?'},\n", - " {u'answers': [{u'answer_start': 70,\n", - " u'text': u'23\\u201316'},\n", - " {u'answer_start': 70,\n", - " u'text': u'23\\u201316'},\n", - " {u'answer_start': 68,\n", - " u'text': u', 23\\u201316,'}],\n", - " u'id': u'56bf36b93aeaaa14008c9562',\n", - " u'question': u'What was the final score of the game between the Broncos and Steelers?'},\n", - " {u'answers': [{u'answer_start': 192,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 192,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 204,\n", - " u'text': u'Patriots'}],\n", - " u'id': u'56bf36b93aeaaa14008c9563',\n", - " u'question': u'Who won Super Bowl XLIX?'},\n", - " {u'answers': [{u'answer_start': 322,\n", - " u'text': u'17'},\n", - " {u'answer_start': 322,\n", - " u'text': u'17 seconds'},\n", - " {u'answer_start': 322,\n", - " u'text': u'17'}],\n", - " u'id': u'56bf36b93aeaaa14008c9564',\n", - " u'question': u'How many seconds were left in the game when the Broncos intercepted the pass that won the game?'},\n", - " {u'answers': [{u'answer_start': 360,\n", - " u'text': u'Manning'},\n", - " {u'answer_start': 360,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56bf36b93aeaaa14008c9565',\n", - " u'question': u\"During the Bronco's playoff games, who did not throw at all?\"},\n", - " {u'answers': [{u'answer_start': 25,\n", - " u'text': u'Pittsburgh Steelers'},\n", - " {u'answer_start': 21,\n", - " u'text': u'the Pittsburgh Steelers'},\n", - " {u'answer_start': 36,\n", - " u'text': u'Steelers'}],\n", - " u'id': u'56d7018a0d65d214001982c2',\n", - " u'question': u'Who did the Broncos beat in the divisional game?'},\n", - " {u'answers': [{u'answer_start': 88,\n", - " u'text': u'11'},\n", - " {u'answer_start': 88,\n", - " u'text': u'11'},\n", - " {u'answer_start': 88,\n", - " u'text': u'11'}],\n", - " u'id': u'56d7018a0d65d214001982c3',\n", - " u'question': u'How many points did the Broncos score in the final three minutes of the Pittsburgh game?'},\n", - " {u'answers': [{u'answer_start': 192,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 192,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 204,\n", - " u'text': u'Patriots'}],\n", - " u'id': u'56d7018a0d65d214001982c5',\n", - " u'question': u'Who did the Broncos defeat in the AFC Championship game?'},\n", - " {u'answers': [{u'answer_start': 25,\n", - " u'text': u'Pittsburgh Steelers'},\n", - " {u'answer_start': 25,\n", - " u'text': u'Pittsburgh Steelers'},\n", - " {u'answer_start': 36,\n", - " u'text': u'Steelers'}],\n", - " u'id': u'56d99f99dc89441400fdb628',\n", - " u'question': u'Who did the Broncos beat to win their division in 2015?'},\n", - " {u'answers': [{u'answer_start': 192,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 192,\n", - " u'text': u'New England Patriots'},\n", - " {u'answer_start': 204,\n", - " u'text': u'Patriots'}],\n", - " u'id': u'56d99f99dc89441400fdb629',\n", - " u'question': u'Who did the Broncos beat tp become the AFC champions?'},\n", - " {u'answers': [{u'answer_start': 322,\n", - " u'text': u'17'},\n", - " {u'answer_start': 322,\n", - " u'text': u'17'},\n", - " {u'answer_start': 322,\n", - " u'text': u'17'}],\n", - " u'id': u'56d99f99dc89441400fdb62c',\n", - " u'question': u'How many seconds were left in the game when the Patriots failed their 2-point conversion?'}]},\n", - " {u'context': u'Carolina suffered a major setback when Thomas Davis, an 11-year veteran who had already overcome three ACL tears in his career, went down with a broken arm in the NFC Championship Game. Despite this, he insisted he would still find a way to play in the Super Bowl. His prediction turned out to be accurate.',\n", - " u'qas': [{u'answers': [{u'answer_start': 39,\n", - " u'text': u'Thomas Davis'},\n", - " {u'answer_start': 39,\n", - " u'text': u'Thomas Davis,'},\n", - " {u'answer_start': 46,\n", - " u'text': u'Davis'}],\n", - " u'id': u'56beb7fd3aeaaa14008c92b5',\n", - " u'question': u'What Carolina player was injured in the NFC Championship Game?'},\n", - " {u'answers': [{u'answer_start': 143,\n", - " u'text': u'a broken arm'},\n", - " {u'answer_start': 143,\n", - " u'text': u'a broken arm'},\n", - " {u'answer_start': 145,\n", - " u'text': u'broken arm'}],\n", - " u'id': u'56beb7fd3aeaaa14008c92b6',\n", - " u'question': u'What injury did Thomas Davis suffer in the NFC Championship?'},\n", - " {u'answers': [{u'answer_start': 97,\n", - " u'text': u'three'},\n", - " {u'answer_start': 97,\n", - " u'text': u'three'},\n", - " {u'answer_start': 97,\n", - " u'text': u'three'}],\n", - " u'id': u'56beb7fd3aeaaa14008c92b7',\n", - " u'question': u'How many times had Thomas Davis torn his ACL in his career?'},\n", - " {u'answers': [{u'answer_start': 56,\n", - " u'text': u'11'},\n", - " {u'answer_start': 56,\n", - " u'text': u'11'},\n", - " {u'answer_start': 56,\n", - " u'text': u'11'}],\n", - " u'id': u'56beb7fd3aeaaa14008c92b8',\n", - " u'question': u'How many years has Thomas Davis played in the NFL?'},\n", - " {u'answers': [{u'answer_start': 103,\n", - " u'text': u'ACL tears'},\n", - " {u'answer_start': 103,\n", - " u'text': u'ACL tears'},\n", - " {u'answer_start': 103,\n", - " u'text': u'ACL'}],\n", - " u'id': u'56bf3a223aeaaa14008c9575',\n", - " u'question': u'What injury did Thomas Davis suffer from repeatedly, for a total of three times, during his career?'},\n", - " {u'answers': [{u'answer_start': 152,\n", - " u'text': u'arm'},\n", - " {u'answer_start': 152,\n", - " u'text': u'arm'},\n", - " {u'answer_start': 152,\n", - " u'text': u'arm'}],\n", - " u'id': u'56bf3a223aeaaa14008c9576',\n", - " u'question': u'What body part did Thomas Davis break during the NFC Championship Game?'},\n", - " {u'answers': [{u'answer_start': 56,\n", - " u'text': u'11'},\n", - " {u'answer_start': 56,\n", - " u'text': u'11'},\n", - " {u'answer_start': 56,\n", - " u'text': u'11'}],\n", - " u'id': u'56bf3a223aeaaa14008c9577',\n", - " u'question': u'How many years has Thomas Davis played in the NFL?'},\n", - " {u'answers': [{u'answer_start': 253,\n", - " u'text': u'Super Bowl'},\n", - " {u'answer_start': 249,\n", - " u'text': u'the Super Bowl'},\n", - " {u'answer_start': 253,\n", - " u'text': u'Super Bowl'}],\n", - " u'id': u'56bf3a223aeaaa14008c9579',\n", - " u'question': u'What game did Thomas Davis say he would play in, despite breaking a bone earlier on?'},\n", - " {u'answers': [{u'answer_start': 97,\n", - " u'text': u'three'},\n", - " {u'answer_start': 97,\n", - " u'text': u'three'},\n", - " {u'answer_start': 97,\n", - " u'text': u'three'}],\n", - " u'id': u'56d703d10d65d214001982d6',\n", - " u'question': u'How many ACL injuries has Thomas Davis had during his career?'},\n", - " {u'answers': [{u'answer_start': 145,\n", - " u'text': u'broken arm'},\n", - " {u'answer_start': 143,\n", - " u'text': u'a broken arm'},\n", - " {u'answer_start': 145,\n", - " u'text': u'broken arm'}],\n", - " u'id': u'56d703d10d65d214001982d7',\n", - " u'question': u'What injury did Thomas Davis suffer during the playoff games?'},\n", - " {u'answers': [{u'answer_start': 56,\n", - " u'text': u'11'},\n", - " {u'answer_start': 56,\n", - " u'text': u'11'},\n", - " {u'answer_start': 56,\n", - " u'text': u'11'}],\n", - " u'id': u'56d703d10d65d214001982d8',\n", - " u'question': u'How many years had Thomas Davis played in the league when he broke his arm during the NFC Championship game?'},\n", - " {u'answers': [{u'answer_start': 39,\n", - " u'text': u'Thomas Davis'},\n", - " {u'answer_start': 39,\n", - " u'text': u'Thomas Davis'},\n", - " {u'answer_start': 46,\n", - " u'text': u'Davis'}],\n", - " u'id': u'56d9a026dc89441400fdb632',\n", - " u'question': u'Who suffered a broken arm in the NFC Championship Game?'}]},\n", - " {u'context': u\"Peyton Manning became the first quarterback ever to lead two different teams to multiple Super Bowls. He is also the oldest quarterback ever to play in a Super Bowl at age 39. The past record was held by John Elway, who led the Broncos to victory in Super Bowl XXXIII at age 38 and is currently Denver's Executive Vice President of Football Operations and General Manager.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 172,\n", - " u'text': u'39'},\n", - " {u'answer_start': 172,\n", - " u'text': u'39'},\n", - " {u'answer_start': 172,\n", - " u'text': u'39'},\n", - " {u'answer_start': 172,\n", - " u'text': u'39'}],\n", - " u'id': u'56beb86b3aeaaa14008c92bd',\n", - " u'question': u'How old was Peyton Manning when he played in Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 204,\n", - " u'text': u'John Elway'},\n", - " {u'answer_start': 204,\n", - " u'text': u'John Elway'},\n", - " {u'answer_start': 209,\n", - " u'text': u'Elway'},\n", - " {u'answer_start': 209,\n", - " u'text': u'Elway'}],\n", - " u'id': u'56beb86b3aeaaa14008c92be',\n", - " u'question': u'Who previously held the record for being the oldest quarterback to play in a Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 275,\n", - " u'text': u'38'},\n", - " {u'answer_start': 275,\n", - " u'text': u'38'},\n", - " {u'answer_start': 275,\n", - " u'text': u'38'},\n", - " {u'answer_start': 275,\n", - " u'text': u'38'}],\n", - " u'id': u'56beb86b3aeaaa14008c92bf',\n", - " u'question': u'How old was John Elway when he played in Super Bowl XXXIII?'},\n", - " {u'answers': [{u'answer_start': 304,\n", - " u'text': u'Executive Vice President of Football Operations and General Manager'},\n", - " {u'answer_start': 304,\n", - " u'text': u'Executive Vice President of Football Operations and General Manager'},\n", - " {u'answer_start': 304,\n", - " u'text': u'Executive Vice President of Football Operations and General Manager.'},\n", - " {u'answer_start': 304,\n", - " u'text': u'Executive Vice President of Football Operations and General Manager.'}],\n", - " u'id': u'56beb86b3aeaaa14008c92c0',\n", - " u'question': u'What role does John Elway currently have in the Broncos franchise?'},\n", - " {u'answers': [{u'answer_start': 228,\n", - " u'text': u'Broncos'},\n", - " {u'answer_start': 224,\n", - " u'text': u'the Broncos'},\n", - " {u'answer_start': 228,\n", - " u'text': u'Broncos'},\n", - " {u'answer_start': 228,\n", - " u'text': u'Broncos'}],\n", - " u'id': u'56beb86b3aeaaa14008c92c1',\n", - " u'question': u'Who did John Elway play for in Super Bowl XXXIII?'},\n", - " {u'answers': [{u'answer_start': 228,\n", - " u'text': u'Broncos'},\n", - " {u'answer_start': 224,\n", - " u'text': u'the Broncos'},\n", - " {u'answer_start': 228,\n", - " u'text': u'Broncos'},\n", - " {u'answer_start': 228,\n", - " u'text': u'Broncos'}],\n", - " u'id': u'56bf3fd53aeaaa14008c9591',\n", - " u'question': u'What team was the winner of Super Bowl XXXIII?'},\n", - " {u'answers': [{u'answer_start': 204,\n", - " u'text': u'John Elway'},\n", - " {u'answer_start': 204,\n", - " u'text': u'John Elway'},\n", - " {u'answer_start': 209,\n", - " u'text': u'Elway'},\n", - " {u'answer_start': 209,\n", - " u'text': u'Elway'}],\n", - " u'id': u'56bf3fd53aeaaa14008c9592',\n", - " u'question': u'Who is the General Manager for the Broncos?'},\n", - " {u'answers': [{u'answer_start': 275,\n", - " u'text': u'38'},\n", - " {u'answer_start': 275,\n", - " u'text': u'38'},\n", - " {u'answer_start': 275,\n", - " u'text': u'38'},\n", - " {u'answer_start': 275,\n", - " u'text': u'38'}],\n", - " u'id': u'56bf3fd53aeaaa14008c9593',\n", - " u'question': u'How old was Elway during his Super Bowl XXXIII win?'},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 7,\n", - " u'text': u'Manning'},\n", - " {u'answer_start': 7,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56bf3fd53aeaaa14008c9594',\n", - " u'question': u'Who has the record of being the oldest quarter back in any Super Bowl game?'},\n", - " {u'answers': [{u'answer_start': 57,\n", - " u'text': u'two'},\n", - " {u'answer_start': 57,\n", - " u'text': u'two'},\n", - " {u'answer_start': 57,\n", - " u'text': u'two'},\n", - " {u'answer_start': 57,\n", - " u'text': u'two'}],\n", - " u'id': u'56bf3fd53aeaaa14008c9595',\n", - " u'question': u'How many teams has Manning played for that reached the Super Bowl, while he was on their team?'},\n", - " {u'answers': [{u'answer_start': 57,\n", - " u'text': u'two'},\n", - " {u'answer_start': 57,\n", - " u'text': u'two'},\n", - " {u'answer_start': 57,\n", - " u'text': u'two'},\n", - " {u'answer_start': 57,\n", - " u'text': u'two'}],\n", - " u'id': u'56d704430d65d214001982de',\n", - " u'question': u'Peyton Manning took how many different teams to the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 7,\n", - " u'text': u'Manning'},\n", - " {u'answer_start': 7,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56d704430d65d214001982e0',\n", - " u'question': u'Who is the oldest quarterback to play in a Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 204,\n", - " u'text': u'John Elway'},\n", - " {u'answer_start': 204,\n", - " u'text': u'John Elway'},\n", - " {u'answer_start': 209,\n", - " u'text': u'Elway'},\n", - " {u'answer_start': 209,\n", - " u'text': u'Elway'}],\n", - " u'id': u'56d704430d65d214001982e1',\n", - " u'question': u'Prior to Manning, who was the oldest quarterback to play in a Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 250,\n", - " u'text': u'Super Bowl XXXIII'},\n", - " {u'answer_start': 250,\n", - " u'text': u'Super Bowl XXXIII'},\n", - " {u'answer_start': 261,\n", - " u'text': u'XXXIII'},\n", - " {u'answer_start': 261,\n", - " u'text': u'XXXIII'}],\n", - " u'id': u'56d704430d65d214001982e2',\n", - " u'question': u'Which Super Bowl did Elway win at 38 years old?'},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 7,\n", - " u'text': u'Manning'},\n", - " {u'answer_start': 7,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56d9a0eadc89441400fdb63e',\n", - " u'question': u'Who was the first quarterback to take two teams to more than one Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 172,\n", - " u'text': u'39.'},\n", - " {u'answer_start': 172,\n", - " u'text': u'39'},\n", - " {u'answer_start': 172,\n", - " u'text': u'39'},\n", - " {u'answer_start': 172,\n", - " u'text': u'39'}],\n", - " u'id': u'56d9a0eadc89441400fdb63f',\n", - " u'question': u'How old was Manning when he played Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 204,\n", - " u'text': u'John Elway'},\n", - " {u'answer_start': 204,\n", - " u'text': u'John Elway'},\n", - " {u'answer_start': 209,\n", - " u'text': u'Elway'},\n", - " {u'answer_start': 209,\n", - " u'text': u'Elway'}],\n", - " u'id': u'56d9a0eadc89441400fdb640',\n", - " u'question': u'What is the name of the quarterback who was 38 in Super Bowl XXXIII?'}]},\n", - " {u'context': u'This was the first Super Bowl to feature a quarterback on both teams who was the #1 pick in their draft classes. Manning was the #1 selection of the 1998 NFL draft, while Newton was picked first in 2011. The matchup also pits the top two picks of the 2011 draft against each other: Newton for Carolina and Von Miller for Denver. Manning and Newton also set the record for the largest age difference between opposing Super Bowl quarterbacks at 13 years and 48 days (Manning was 39, Newton was 26).',\n", - " u'qas': [{u'answers': [{u'answer_start': 149,\n", - " u'text': u'1998'},\n", - " {u'answer_start': 149,\n", - " u'text': u'1998'},\n", - " {u'answer_start': 149,\n", - " u'text': u'1998'}],\n", - " u'id': u'56beb90c3aeaaa14008c92c7',\n", - " u'question': u'When was Manning picked #1 in the NFL Draft?'},\n", - " {u'answers': [{u'answer_start': 198,\n", - " u'text': u'2011'},\n", - " {u'answer_start': 198,\n", - " u'text': u'2011'},\n", - " {u'answer_start': 198,\n", - " u'text': u'2011'}],\n", - " u'id': u'56beb90c3aeaaa14008c92c8',\n", - " u'question': u'When was Newton the first selection in the NFL Draft?'},\n", - " {u'answers': [{u'answer_start': 492,\n", - " u'text': u'26'},\n", - " {u'answer_start': 492,\n", - " u'text': u'26'},\n", - " {u'answer_start': 492,\n", - " u'text': u'26'}],\n", - " u'id': u'56beb90c3aeaaa14008c92c9',\n", - " u'question': u'How old was Newton during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 443,\n", - " u'text': u'13 years and 48 days'},\n", - " {u'answer_start': 443,\n", - " u'text': u'13 years and 48 days'},\n", - " {u'answer_start': 443,\n", - " u'text': u'13 years'}],\n", - " u'id': u'56beb90c3aeaaa14008c92ca',\n", - " u'question': u'How much older was Manning than Newton during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 306,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 306,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 310,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56beb90c3aeaaa14008c92cb',\n", - " u'question': u'Who was the #2 pick in the 2011 NFL Draft?'},\n", - " {u'answers': [{u'answer_start': 113,\n", - " u'text': u'Manning'},\n", - " {u'answer_start': 113,\n", - " u'text': u'Manning'},\n", - " {u'answer_start': 113,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56bf41013aeaaa14008c959b',\n", - " u'question': u'Who was considered to be the first choice in the NFL draft of 1998?'},\n", - " {u'answers': [{u'answer_start': 171,\n", - " u'text': u'Newton'},\n", - " {u'answer_start': 171,\n", - " u'text': u'Newton'},\n", - " {u'answer_start': 171,\n", - " u'text': u'Newton'}],\n", - " u'id': u'56bf41013aeaaa14008c959c',\n", - " u'question': u'In 2011, who was the first player to be chosen in the NFL draft?'},\n", - " {u'answers': [{u'answer_start': 492,\n", - " u'text': u'26'},\n", - " {u'answer_start': 492,\n", - " u'text': u'26'},\n", - " {u'answer_start': 492,\n", - " u'text': u'26'}],\n", - " u'id': u'56bf41013aeaaa14008c959d',\n", - " u'question': u'How old was Newton during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 43,\n", - " u'text': u'quarterback'},\n", - " {u'answer_start': 427,\n", - " u'text': u'quarterbacks'},\n", - " {u'answer_start': 43,\n", - " u'text': u'quarterback'}],\n", - " u'id': u'56bf41013aeaaa14008c959e',\n", - " u'question': u'What position did Newton play during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 149,\n", - " u'text': u'1998'},\n", - " {u'answer_start': 149,\n", - " u'text': u'1998'},\n", - " {u'answer_start': 149,\n", - " u'text': u'1998'}],\n", - " u'id': u'56d7096b0d65d214001982fc',\n", - " u'question': u'Manning was the number one pick in which draft?'},\n", - " {u'answers': [{u'answer_start': 198,\n", - " u'text': u'2011'},\n", - " {u'answer_start': 198,\n", - " u'text': u'2011'},\n", - " {u'answer_start': 198,\n", - " u'text': u'2011'}],\n", - " u'id': u'56d7096b0d65d214001982fd',\n", - " u'question': u'Newton was the number one pick in which draft?'},\n", - " {u'answers': [{u'answer_start': 306,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 306,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 310,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56d7096b0d65d214001982fe',\n", - " u'question': u'Who was the number two draft pick for 2011?'},\n", - " {u'answers': [{u'answer_start': 198,\n", - " u'text': u'2011.'},\n", - " {u'answer_start': 198,\n", - " u'text': u'2011'},\n", - " {u'answer_start': 198,\n", - " u'text': u'2011'}],\n", - " u'id': u'56d9a199dc89441400fdb647',\n", - " u'question': u'What year did Newton hold the number one draft pick position?'},\n", - " {u'answers': [{u'answer_start': 492,\n", - " u'text': u'26'},\n", - " {u'answer_start': 492,\n", - " u'text': u'26'},\n", - " {u'answer_start': 492,\n", - " u'text': u'26'}],\n", - " u'id': u'56d9a199dc89441400fdb648',\n", - " u'question': u'How old was Newton in Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 443,\n", - " u'text': u'13 years and 48 days'},\n", - " {u'answer_start': 443,\n", - " u'text': u'13 years and 48 days'},\n", - " {u'answer_start': 443,\n", - " u'text': u'13 years and 48 days'}],\n", - " u'id': u'56d9a199dc89441400fdb649',\n", - " u'question': u'What was the age difference between Newton and Manning in Super Bowl 50?'}]},\n", - " {u'context': u\"With Rivera having been a linebacker with the Chicago Bears in Super Bowl XX, and Kubiak replacing Elway at the end of the Broncos' defeats in Super Bowls XXI and XXIV, this will be the first Super Bowl in which both head coaches played in the game themselves.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 63,\n", - " u'text': u'Super Bowl XX'},\n", - " {u'answer_start': 63,\n", - " u'text': u'Super Bowl XX'},\n", - " {u'answer_start': 74,\n", - " u'text': u'XX'}],\n", - " u'id': u'56beb97c3aeaaa14008c92db',\n", - " u'question': u'In what Super Bowl did Rivera play?'},\n", - " {u'answers': [{u'answer_start': 46,\n", - " u'text': u'Chicago Bears'},\n", - " {u'answer_start': 42,\n", - " u'text': u'the Chicago Bears'},\n", - " {u'answer_start': 54,\n", - " u'text': u'Bears'}],\n", - " u'id': u'56beb97c3aeaaa14008c92dc',\n", - " u'question': u'What team did Rivera play for in Super Bowl XX?'},\n", - " {u'answers': [{u'answer_start': 26,\n", - " u'text': u'linebacker'},\n", - " {u'answer_start': 26,\n", - " u'text': u'linebacker'},\n", - " {u'answer_start': 26,\n", - " u'text': u'linebacker'}],\n", - " u'id': u'56beb97c3aeaaa14008c92dd',\n", - " u'question': u'What position did Rivera play in Super Bowl XX?'},\n", - " {u'answers': [{u'answer_start': 99,\n", - " u'text': u'Elway'},\n", - " {u'answer_start': 99,\n", - " u'text': u'Elway'},\n", - " {u'answer_start': 99,\n", - " u'text': u'Elway'}],\n", - " u'id': u'56beb97c3aeaaa14008c92de',\n", - " u'question': u'Who was replaced by Kubiak in Super Bowl XXIV?'},\n", - " {u'answers': [{u'answer_start': 123,\n", - " u'text': u'Broncos'},\n", - " {u'answer_start': 119,\n", - " u'text': u'the Broncos'},\n", - " {u'answer_start': 123,\n", - " u'text': u'Broncos'}],\n", - " u'id': u'56beb97c3aeaaa14008c92df',\n", - " u'question': u'What team did Kubiak play for in Super Bowl XXI?'},\n", - " {u'answers': [{u'answer_start': 26,\n", - " u'text': u'linebacker'},\n", - " {u'answer_start': 26,\n", - " u'text': u'linebacker'},\n", - " {u'answer_start': 26,\n", - " u'text': u'linebacker'}],\n", - " u'id': u'56bf6c3e3aeaaa14008c9615',\n", - " u'question': u'What position did Rivera play in Super Bowl XX? '},\n", - " {u'answers': [{u'answer_start': 99,\n", - " u'text': u'Elway'},\n", - " {u'answer_start': 99,\n", - " u'text': u'Elway'},\n", - " {u'answer_start': 99,\n", - " u'text': u'Elway'}],\n", - " u'id': u'56bf6c3e3aeaaa14008c9617',\n", - " u'question': u'Who did Kubiak take the place of after Super Bowl XXIV?'},\n", - " {u'answers': [{u'answer_start': 5,\n", - " u'text': u'Rivera'},\n", - " {u'answer_start': 5,\n", - " u'text': u'Rivera'},\n", - " {u'answer_start': 5,\n", - " u'text': u'Rivera'}],\n", - " u'id': u'56d709ef0d65d21400198306',\n", - " u'question': u'Who was the Panthers head coach for the 2015 season?'},\n", - " {u'answers': [{u'answer_start': 63,\n", - " u'text': u'Super Bowl XX'},\n", - " {u'answer_start': 63,\n", - " u'text': u'Super Bowl XX'},\n", - " {u'answer_start': 74,\n", - " u'text': u'XX'}],\n", - " u'id': u'56d709ef0d65d21400198307',\n", - " u'question': u'What Super Bowl did Rivera play in as a player?'}]},\n", - " {u'context': u'Concerns were raised over whether Levi\\'s Stadium\\'s field was of a high enough quality to host a Super Bowl; during the inaugural season, the field had to be re-sodded multiple times due to various issues, and during a week 6 game earlier in the 2015 season, a portion of the turf collapsed under Baltimore Ravens kicker Justin Tucker, causing him to slip and miss a field goal, although the field has not had any major issues since. As is customary for Super Bowl games played at natural grass stadiums, the NFL re-sodded the field with a new playing surface; a hybrid Bermuda 419 turf. NFL and Atlanta Braves field director Ed Mangan stated that the field was in \"great shape\" for gameday. However, the turf showed problem throughout the game, with a number of players needing to change their cleats during the game and player slipping during plays all throughout the game.',\n", - " u'qas': [{u'answers': [{u'answer_start': 320,\n", - " u'text': u'Justin Tucker'},\n", - " {u'answer_start': 320,\n", - " u'text': u'Justin Tucker'},\n", - " {u'answer_start': 327,\n", - " u'text': u'Tucker'}],\n", - " u'id': u'56beba103aeaaa14008c92e5',\n", - " u'question': u\"Who slipped on the Levi's Stadium turf in week 6 of the 2015 NFL season?\"},\n", - " {u'answers': [{u'answer_start': 569,\n", - " u'text': u'Bermuda 419'},\n", - " {u'answer_start': 569,\n", - " u'text': u'Bermuda 419'},\n", - " {u'answer_start': 562,\n", - " u'text': u'hybrid Bermuda 419'}],\n", - " u'id': u'56beba103aeaaa14008c92e6',\n", - " u'question': u\"What is the name of the turf used in Levi's Stadium for the Super Bowl?\"},\n", - " {u'answers': [{u'answer_start': 625,\n", - " u'text': u'Ed Mangan'},\n", - " {u'answer_start': 625,\n", - " u'text': u'Ed Mangan'},\n", - " {u'answer_start': 628,\n", - " u'text': u'Mangan'}],\n", - " u'id': u'56beba103aeaaa14008c92e7',\n", - " u'question': u'Who is the field director of the NFL?'},\n", - " {u'answers': [{u'answer_start': 296,\n", - " u'text': u'Baltimore Ravens'},\n", - " {u'answer_start': 296,\n", - " u'text': u'Baltimore Ravens'},\n", - " {u'answer_start': 306,\n", - " u'text': u'Ravens'}],\n", - " u'id': u'56beba103aeaaa14008c92e8',\n", - " u'question': u'What team did Justin Tucker play for?'},\n", - " {u'answers': [{u'answer_start': 313,\n", - " u'text': u'kicker'},\n", - " {u'answer_start': 313,\n", - " u'text': u'kicker'},\n", - " {u'answer_start': 313,\n", - " u'text': u'kicker'}],\n", - " u'id': u'56beba103aeaaa14008c92e9',\n", - " u'question': u'What position did Justin Tucker play?'},\n", - " {u'answers': [{u'answer_start': 320,\n", - " u'text': u'Justin Tucker'},\n", - " {u'answer_start': 320,\n", - " u'text': u'Justin Tucker'},\n", - " {u'answer_start': 327,\n", - " u'text': u'Tucker'}],\n", - " u'id': u'56bf467d3aeaaa14008c95a5',\n", - " u'question': u\"Which player did the turf collapse under during a game in the Levi's Stadium?\"},\n", - " {u'answers': [{u'answer_start': 313,\n", - " u'text': u'kicker'},\n", - " {u'answer_start': 313,\n", - " u'text': u'kicker'},\n", - " {u'answer_start': 313,\n", - " u'text': u'kicker'}],\n", - " u'id': u'56bf467d3aeaaa14008c95a6',\n", - " u'question': u'What position does Tucker play for with the Ravens?'},\n", - " {u'answers': [{u'answer_start': 562,\n", - " u'text': u'hybrid Bermuda 419 turf'},\n", - " {u'answer_start': 569,\n", - " u'text': u'Bermuda 419 turf'},\n", - " {u'answer_start': 562,\n", - " u'text': u'hybrid Bermuda 419'}],\n", - " u'id': u'56bf467d3aeaaa14008c95a7',\n", - " u'question': u\"What was used to sod the Levi's Stadium for Super Bowl 50? \"},\n", - " {u'answers': [{u'answer_start': 320,\n", - " u'text': u'Justin Tucker'},\n", - " {u'answer_start': 320,\n", - " u'text': u'Justin Tucker'},\n", - " {u'answer_start': 327,\n", - " u'text': u'Tucker'}],\n", - " u'id': u'56d70adc0d65d21400198311',\n", - " u'question': u'Which kicker had a portion of the turf collapse, causing him to miss a field goal?'},\n", - " {u'answers': [{u'answer_start': 537,\n", - " u'text': u'a new playing surface'},\n", - " {u'answer_start': 512,\n", - " u'text': u're-sodded'},\n", - " {u'answer_start': 157,\n", - " u'text': u're-sodded'}],\n", - " u'id': u'56d70adc0d65d21400198312',\n", - " u'question': u\"What did the NFL do to the playing field at Levi's Stadium before the Super Bowl?\"},\n", - " {u'answers': [{u'answer_start': 560,\n", - " u'text': u'a hybrid Bermuda 419 turf.'},\n", - " {u'answer_start': 569,\n", - " u'text': u'Bermuda 419 turf'},\n", - " {u'answer_start': 562,\n", - " u'text': u'hybrid Bermuda 419 turf'}],\n", - " u'id': u'56d70adc0d65d21400198313',\n", - " u'question': u\"What covered the new field at Levi's Stadium?\"},\n", - " {u'answers': [{u'answer_start': 788,\n", - " u'text': u'their cleats'},\n", - " {u'answer_start': 794,\n", - " u'text': u'cleats'},\n", - " {u'answer_start': 794,\n", - " u'text': u'cleats'}],\n", - " u'id': u'56d70adc0d65d21400198314',\n", - " u'question': u'What did a lot of players need to change during Super Bowl 50 because of the condition of the field?'},\n", - " {u'answers': [{u'answer_start': 320,\n", - " u'text': u'Justin Tucker'},\n", - " {u'answer_start': 327,\n", - " u'text': u'Tucker'}],\n", - " u'id': u'56d9a419dc89441400fdb678',\n", - " u'question': u\"What player did the field problem at Levi's stadium affect for a field goal in week six of the regular season?\"},\n", - " {u'answers': [{u'answer_start': 480,\n", - " u'text': u'natural grass'},\n", - " {u'answer_start': 480,\n", - " u'text': u'natural grass'},\n", - " {u'answer_start': 488,\n", - " u'text': u'grass'}],\n", - " u'id': u'56d9a419dc89441400fdb679',\n", - " u'question': u'What field type is usually chosen for Super Bowl games?'}]},\n", - " {u'context': u'As the designated home team in the annual rotation between AFC and NFC teams, the Broncos elected to wear their road white jerseys with matching white pants. Elway stated, \"We\\'ve had Super Bowl success in our white uniforms.\" The Broncos last wore matching white jerseys and pants in the Super Bowl in Super Bowl XXXIII, Elway\\'s last game as Denver QB, when they defeated the Atlanta Falcons 34\\u201319. In their only other Super Bowl win in Super Bowl XXXII, Denver wore blue jerseys, which was their primary color at the time. They also lost Super Bowl XXI when they wore white jerseys, but they are 0-4 in Super Bowls when wearing orange jerseys, losing in Super Bowl XII, XXII, XXIV, and XLVIII. The only other AFC champion team to have worn white as the designated home team in the Super Bowl was the Pittsburgh Steelers; they defeated the Seattle Seahawks 21\\u201310 in Super Bowl XL 10 seasons prior. The Broncos\\' decision to wear white meant the Panthers would wear their standard home uniform: black jerseys with silver pants.',\n", - " u'qas': [{u'answers': [{u'answer_start': 82,\n", - " u'text': u'Broncos'},\n", - " {u'answer_start': 78,\n", - " u'text': u'the Broncos'},\n", - " {u'answer_start': 82,\n", - " u'text': u'Broncos'}],\n", - " u'id': u'56bebad93aeaaa14008c92f9',\n", - " u'question': u'Who was designated as the home team in the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 392,\n", - " u'text': u'34\\u201319'},\n", - " {u'answer_start': 392,\n", - " u'text': u'34\\u201319'},\n", - " {u'answer_start': 392,\n", - " u'text': u'34\\u201319'}],\n", - " u'id': u'56bebad93aeaaa14008c92fa',\n", - " u'question': u'What was the final score of Super Bowl XXXIII?'},\n", - " {u'answers': [{u'answer_start': 376,\n", - " u'text': u'Atlanta Falcons'},\n", - " {u'answer_start': 372,\n", - " u'text': u'the Atlanta Falcons'},\n", - " {u'answer_start': 384,\n", - " u'text': u'Falcons'}],\n", - " u'id': u'56bebad93aeaaa14008c92fb',\n", - " u'question': u'What team lost Super Bowl XXXIII?'},\n", - " {u'answers': [{u'answer_start': 117,\n", - " u'text': u'white'},\n", - " {u'answer_start': 145,\n", - " u'text': u'white'},\n", - " {u'answer_start': 145,\n", - " u'text': u'white'}],\n", - " u'id': u'56bebad93aeaaa14008c92fc',\n", - " u'question': u'What color pants did the Broncos wear in Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 302,\n", - " u'text': u'Super Bowl XXXIII'},\n", - " {u'answer_start': 302,\n", - " u'text': u'Super Bowl XXXIII'},\n", - " {u'answer_start': 302,\n", - " u'text': u'Super Bowl XXXIII'}],\n", - " u'id': u'56bebad93aeaaa14008c92fd',\n", - " u'question': u\"What was Elway's final game as the Denver quarterback?\"},\n", - " {u'answers': [{u'answer_start': 302,\n", - " u'text': u'Super Bowl XXXIII'},\n", - " {u'answer_start': 302,\n", - " u'text': u'Super Bowl XXXIII'},\n", - " {u'answer_start': 302,\n", - " u'text': u'Super Bowl XXXIII'}],\n", - " u'id': u'56bf48cc3aeaaa14008c95ab',\n", - " u'question': u'What was the last game that Elway played as a quarterback for Denver?'},\n", - " {u'answers': [{u'answer_start': 392,\n", - " u'text': u'34\\u201319'},\n", - " {u'answer_start': 392,\n", - " u'text': u'34\\u201319'},\n", - " {u'answer_start': 392,\n", - " u'text': u'34\\u201319'}],\n", - " u'id': u'56bf48cc3aeaaa14008c95ac',\n", - " u'question': u'What was the final score for Super Bowl XXXIII?'},\n", - " {u'answers': [{u'answer_start': 376,\n", - " u'text': u'Atlanta Falcons'},\n", - " {u'answer_start': 372,\n", - " u'text': u'the Atlanta Falcons'},\n", - " {u'answer_start': 384,\n", - " u'text': u'Falcons'}],\n", - " u'id': u'56bf48cc3aeaaa14008c95ae',\n", - " u'question': u'What team did the Denver Broncos play in Super Bowl XXXIII?'},\n", - " {u'answers': [{u'answer_start': 117,\n", - " u'text': u'white'},\n", - " {u'answer_start': 248,\n", - " u'text': u'matching white jerseys'},\n", - " {u'answer_start': 209,\n", - " u'text': u'white'}],\n", - " u'id': u'56bf48cc3aeaaa14008c95af',\n", - " u'question': u\"What color were the Bronco's uniforms in Super Bowl 50?\"},\n", - " {u'answers': [{u'answer_start': 112,\n", - " u'text': u'road white jerseys'},\n", - " {u'answer_start': 136,\n", - " u'text': u'matching white'},\n", - " {u'answer_start': 117,\n", - " u'text': u'white'}],\n", - " u'id': u'56d70ccc0d65d2140019831d',\n", - " u'question': u'What jersey did the Broncos wear for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 801,\n", - " u'text': u'Pittsburgh Steelers'},\n", - " {u'answer_start': 797,\n", - " u'text': u'the Pittsburgh Steelers'},\n", - " {u'answer_start': 812,\n", - " u'text': u'Steelers'}],\n", - " u'id': u'56d70ccc0d65d2140019831f',\n", - " u'question': u'Who is the only other AFC Champion team to wear road jerseys in a Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 302,\n", - " u'text': u'Super Bowl XXXIII'},\n", - " {u'answer_start': 302,\n", - " u'text': u'Super Bowl XXXIII'},\n", - " {u'answer_start': 313,\n", - " u'text': u'XXXIII'}],\n", - " u'id': u'56d9a637dc89441400fdb697',\n", - " u'question': u'What was th elast Super Bowl where the Broncos wore white uniforms?'},\n", - " {u'answers': [{u'answer_start': 467,\n", - " u'text': u'blue'},\n", - " {u'answer_start': 467,\n", - " u'text': u'blue'},\n", - " {u'answer_start': 209,\n", - " u'text': u'white'}],\n", - " u'id': u'56d9a637dc89441400fdb698',\n", - " u'question': u'What was the color of the jerseys in Super Bowl XXXII when Elway was quarterback?'},\n", - " {u'answers': [{u'answer_start': 629,\n", - " u'text': u'orange'},\n", - " {u'answer_start': 629,\n", - " u'text': u'orange'},\n", - " {u'answer_start': 629,\n", - " u'text': u'orange'}],\n", - " u'id': u'56d9a637dc89441400fdb699',\n", - " u'question': u'What color jersey has Denver 0-4?'},\n", - " {u'answers': [{u'answer_start': 993,\n", - " u'text': u'black jerseys with silver pants.'},\n", - " {u'answer_start': 993,\n", - " u'text': u'black jerseys with silver pants'},\n", - " {u'answer_start': 993,\n", - " u'text': u'black'}],\n", - " u'id': u'56d9a637dc89441400fdb69a',\n", - " u'question': u'Since Denver chose white, what colors did Carolina wear in Super Bowl 50?'}]},\n", - " {u'context': u'The Panthers used the San Jose State practice facility and stayed at the San Jose Marriott. The Broncos practiced at Stanford University and stayed at the Santa Clara Marriott.',\n", - " u'qas': [{u'answers': [{u'answer_start': 22,\n", - " u'text': u'San Jose State'},\n", - " {u'answer_start': 22,\n", - " u'text': u'San Jose State'},\n", - " {u'answer_start': 22,\n", - " u'text': u'San Jose State'}],\n", - " u'id': u'56bebb293aeaaa14008c9303',\n", - " u'question': u\"At what university's facility did the Panthers practice?\"},\n", - " {u'answers': [{u'answer_start': 117,\n", - " u'text': u'Stanford University'},\n", - " {u'answer_start': 117,\n", - " u'text': u'Stanford University'},\n", - " {u'answer_start': 117,\n", - " u'text': u'Stanford'}],\n", - " u'id': u'56bebb293aeaaa14008c9304',\n", - " u'question': u\"At what university's facility did the Broncos practice?\"},\n", - " {u'answers': [{u'answer_start': 22,\n", - " u'text': u'San Jose'},\n", - " {u'answer_start': 73,\n", - " u'text': u'San Jose'},\n", - " {u'answer_start': 73,\n", - " u'text': u'San Jose'}],\n", - " u'id': u'56bebb293aeaaa14008c9305',\n", - " u'question': u\"In what city's Marriott did the Panthers stay?\"},\n", - " {u'answers': [{u'answer_start': 155,\n", - " u'text': u'Santa Clara'},\n", - " {u'answer_start': 155,\n", - " u'text': u'Santa Clara'},\n", - " {u'answer_start': 155,\n", - " u'text': u'Santa Clara'}],\n", - " u'id': u'56bebb293aeaaa14008c9306',\n", - " u'question': u\"In what city's Marriott did the Broncos stay?\"},\n", - " {u'answers': [{u'answer_start': 73,\n", - " u'text': u'San Jose Marriott'},\n", - " {u'answer_start': 69,\n", - " u'text': u'the San Jose Marriott'},\n", - " {u'answer_start': 22,\n", - " u'text': u'San Jose State practice facility and stayed at the San Jose Marriott.'}],\n", - " u'id': u'56bf49993aeaaa14008c95b5',\n", - " u'question': u'What hotel did the Panthers stay in during Super Bowl 50? '},\n", - " {u'answers': [{u'answer_start': 155,\n", - " u'text': u'Santa Clara Marriott'},\n", - " {u'answer_start': 69,\n", - " u'text': u'the San Jose Marriott'},\n", - " {u'answer_start': 82,\n", - " u'text': u'Marriott'}],\n", - " u'id': u'56bf49993aeaaa14008c95b6',\n", - " u'question': u'What was the name of the hotel the Panthers chose to stay in during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 22,\n", - " u'text': u'San Jose State practice facility'},\n", - " {u'answer_start': 18,\n", - " u'text': u'the San Jose State practice facility'},\n", - " {u'answer_start': 22,\n", - " u'text': u'San Jose State'}],\n", - " u'id': u'56bf49993aeaaa14008c95b7',\n", - " u'question': u'Where did the Panthers practice at for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 117,\n", - " u'text': u'Stanford University'},\n", - " {u'answer_start': 114,\n", - " u'text': u'at Stanford University'},\n", - " {u'answer_start': 117,\n", - " u'text': u'Stanford'}],\n", - " u'id': u'56bf49993aeaaa14008c95b8',\n", - " u'question': u'Where did the Broncos practice at for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 22,\n", - " u'text': u'San Jose State practice facility'},\n", - " {u'answer_start': 18,\n", - " u'text': u'the San Jose State practice facility'},\n", - " {u'answer_start': 22,\n", - " u'text': u'San Jose State'}],\n", - " u'id': u'56d70d240d65d21400198326',\n", - " u'question': u'Where did the Panthers practice for the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 73,\n", - " u'text': u'San Jose Marriott.'},\n", - " {u'answer_start': 69,\n", - " u'text': u'the San Jose Marriott'},\n", - " {u'answer_start': 73,\n", - " u'text': u'San Jose Marriott'}],\n", - " u'id': u'56d70d240d65d21400198327',\n", - " u'question': u'Which hotel did the Panthers stay at for the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 117,\n", - " u'text': u'Stanford University'},\n", - " {u'answer_start': 114,\n", - " u'text': u'at Stanford University'},\n", - " {u'answer_start': 117,\n", - " u'text': u'Stanford'}],\n", - " u'id': u'56d70d240d65d21400198328',\n", - " u'question': u'Where did the Broncos practice for the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 155,\n", - " u'text': u'Santa Clara Marriott.'},\n", - " {u'answer_start': 151,\n", - " u'text': u'the Santa Clara Marriott'},\n", - " {u'answer_start': 155,\n", - " u'text': u'Santa Clara Marriott.'}],\n", - " u'id': u'56d70d240d65d21400198329',\n", - " u'question': u'Which hotel did the Broncos use for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 22,\n", - " u'text': u'San Jose'},\n", - " {u'answer_start': 18,\n", - " u'text': u'the San Jose State practice facility'},\n", - " {u'answer_start': 22,\n", - " u'text': u'San Jose State'}],\n", - " u'id': u'56d9a6b4dc89441400fdb6a0',\n", - " u'question': u'Where was the practice place the Panthers used for the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 73,\n", - " u'text': u'San Jose Marriott.'},\n", - " {u'answer_start': 69,\n", - " u'text': u'the San Jose Marriott'},\n", - " {u'answer_start': 73,\n", - " u'text': u'San Jose Marriott.'}],\n", - " u'id': u'56d9a6b4dc89441400fdb6a1',\n", - " u'question': u'Where hotel did the Panthers stay at?'},\n", - " {u'answers': [{u'answer_start': 117,\n", - " u'text': u'Stanford University'},\n", - " {u'answer_start': 114,\n", - " u'text': u'at Stanford University'},\n", - " {u'answer_start': 117,\n", - " u'text': u'Stanford'}],\n", - " u'id': u'56d9a6b4dc89441400fdb6a2',\n", - " u'question': u'Where did the Broncos practice for the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 155,\n", - " u'text': u'Santa Clara Marriott.'},\n", - " {u'answer_start': 151,\n", - " u'text': u'the Santa Clara Marriott'},\n", - " {u'answer_start': 155,\n", - " u'text': u'Santa Clara Marriott.'}],\n", - " u'id': u'56d9a6b4dc89441400fdb6a3',\n", - " u'question': u'Where did the Broncos stay at for Super Bowl 50?'}]},\n", - " {u'context': u'On June 4, 2014, the NFL announced that the practice of branding Super Bowl games with Roman numerals, a practice established at Super Bowl V, would be temporarily suspended, and that the game would be named using Arabic numerals as Super Bowl 50 as opposed to Super Bowl L. The use of Roman numerals will be reinstated for Super Bowl LI. Jaime Weston, the league\\'s vice president of brand and creative, explained that a primary reason for the change was the difficulty of designing an aesthetically pleasing logo with the letter \"L\" using the standardized logo template introduced at Super Bowl XLV. The logo also deviates from the template by featuring large numerals, colored in gold, behind the Vince Lombardi Trophy, instead of underneath and in silver as in the standard logo.',\n", - " u'qas': [{u'answers': [{u'answer_start': 3,\n", - " u'text': u'June 4, 2014'},\n", - " {u'answer_start': 0,\n", - " u'text': u'On June 4, 2014'},\n", - " {u'answer_start': 11,\n", - " u'text': u'2014'}],\n", - " u'id': u'56bebbbf3aeaaa14008c9315',\n", - " u'question': u'When did the NFL announce the suspension of using Roman numerals to brand the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 129,\n", - " u'text': u'Super Bowl V'},\n", - " {u'answer_start': 129,\n", - " u'text': u'Super Bowl V'},\n", - " {u'answer_start': 140,\n", - " u'text': u'V'}],\n", - " u'id': u'56bebbbf3aeaaa14008c9316',\n", - " u'question': u'What was the first Super Bowl branded with Roman numerals?'},\n", - " {u'answers': [{u'answer_start': 339,\n", - " u'text': u'Jaime Weston'},\n", - " {u'answer_start': 339,\n", - " u'text': u'Jaime Weston'},\n", - " {u'answer_start': 345,\n", - " u'text': u'Weston'}],\n", - " u'id': u'56bebbbf3aeaaa14008c9317',\n", - " u'question': u\"Who is the NFL's vice president of brand and creative?\"},\n", - " {u'answers': [{u'answer_start': 585,\n", - " u'text': u'Super Bowl XLV'},\n", - " {u'answer_start': 585,\n", - " u'text': u'Super Bowl XLV'},\n", - " {u'answer_start': 140,\n", - " u'text': u'V'}],\n", - " u'id': u'56bebbbf3aeaaa14008c9318',\n", - " u'question': u'What was the first Super Bowl to use the standardized logo template?'},\n", - " {u'answers': [{u'answer_start': 699,\n", - " u'text': u'Vince Lombardi'},\n", - " {u'answer_start': 699,\n", - " u'text': u'Vince Lombardi Trophy'},\n", - " {u'answer_start': 705,\n", - " u'text': u'Lombardi'}],\n", - " u'id': u'56bebbbf3aeaaa14008c9319',\n", - " u'question': u'Who is the trophy featured on the logo named for?'},\n", - " {u'answers': [{u'answer_start': 11,\n", - " u'text': u'2014'},\n", - " {u'answer_start': 11,\n", - " u'text': u'2014'},\n", - " {u'answer_start': 11,\n", - " u'text': u'2014'}],\n", - " u'id': u'56bf4e1b3aeaaa14008c95bd',\n", - " u'question': u'In what year was it declared that no Roman numerals would be used in the name of the 50th Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 324,\n", - " u'text': u'Super Bowl LI'},\n", - " {u'answer_start': 335,\n", - " u'text': u'LI'},\n", - " {u'answer_start': 335,\n", - " u'text': u'LI'}],\n", - " u'id': u'56bf4e1b3aeaaa14008c95be',\n", - " u'question': u'Which Super Bowl, after the 50th one, will begin have Roman numerals in the title again?'},\n", - " {u'answers': [{u'answer_start': 23,\n", - " u'text': u'L'},\n", - " {u'answer_start': 272,\n", - " u'text': u'L'},\n", - " {u'answer_start': 272,\n", - " u'text': u'L'}],\n", - " u'id': u'56bf4e1b3aeaaa14008c95c0',\n", - " u'question': u'If Roman numerals were used in the naming of the 50th Super Bowl, which one would have been used?'},\n", - " {u'answers': [{u'answer_start': 682,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 682,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 682,\n", - " u'text': u'gold'}],\n", - " u'id': u'56bf4e1b3aeaaa14008c95c1',\n", - " u'question': u'What color are the numbers in the Super Bowl 50 logo? '},\n", - " {u'answers': [{u'answer_start': 3,\n", - " u'text': u'June 4, 2014'},\n", - " {u'answer_start': 0,\n", - " u'text': u'On June 4, 2014'},\n", - " {u'answer_start': 11,\n", - " u'text': u'2014'}],\n", - " u'id': u'56d70daa0d65d21400198332',\n", - " u'question': u'When did the NFL announce that it would not use Roman numbers to designate the Super Bowl number?'},\n", - " {u'answers': [{u'answer_start': 214,\n", - " u'text': u'Arabic numerals'},\n", - " {u'answer_start': 214,\n", - " u'text': u'Arabic numerals'},\n", - " {u'answer_start': 214,\n", - " u'text': u'Arabic'}],\n", - " u'id': u'56d70daa0d65d21400198333',\n", - " u'question': u'Rather than Roman numerals, what did the NFL decide to use?'},\n", - " {u'answers': [{u'answer_start': 272,\n", - " u'text': u'L.'},\n", - " {u'answer_start': 272,\n", - " u'text': u'L'},\n", - " {u'answer_start': 272,\n", - " u'text': u'L'}],\n", - " u'id': u'56d70daa0d65d21400198334',\n", - " u'question': u'What is the Roman numeral for 50?'},\n", - " {u'answers': [{u'answer_start': 682,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 682,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 682,\n", - " u'text': u'gold'}],\n", - " u'id': u'56d70daa0d65d21400198335',\n", - " u'question': u'Normally silver, what color was used for the number 50?'},\n", - " {u'answers': [{u'answer_start': 324,\n", - " u'text': u'Super Bowl LI.'},\n", - " {u'answer_start': 324,\n", - " u'text': u'Super Bowl LI'},\n", - " {u'answer_start': 324,\n", - " u'text': u'Super Bowl LI'}],\n", - " u'id': u'56d70daa0d65d21400198336',\n", - " u'question': u'When will Roman numerals be used again to denote the Super Bowl number?'},\n", - " {u'answers': [{u'answer_start': 214,\n", - " u'text': u'Arabic'},\n", - " {u'answer_start': 214,\n", - " u'text': u'Arabic numerals'},\n", - " {u'answer_start': 214,\n", - " u'text': u'Arabic'}],\n", - " u'id': u'56d9a7addc89441400fdb6a8',\n", - " u'question': u'What type of numeral did the latest Super Bowl use to designate the game number?'},\n", - " {u'answers': [{u'answer_start': 335,\n", - " u'text': u'LI.'},\n", - " {u'answer_start': 335,\n", - " u'text': u'LI'},\n", - " {u'answer_start': 335,\n", - " u'text': u'LI'}],\n", - " u'id': u'56d9a7addc89441400fdb6ab',\n", - " u'question': u'What is the Roman numeral going to be for Super Bowl 51?'}]},\n", - " {u'context': u'Various gold-themed promotions and initiatives were held throughout the 2015 NFL season to tie into the \"Golden Super Bowl\"; gold-tinted logos were implemented across the NFL\\'s properties and painted on fields, the numbering of the 50-yard line on fields was colored gold, and beginning on week 7, all sideline jackets and hats featured gold-trimmed logos. Gold footballs were given to each high school that has had a player or coach appear in the Super Bowl, and \"homecoming\" events were also held by Super Bowl-winning teams at games.',\n", - " u'qas': [{u'answers': [{u'answer_start': 8,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 8,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 267,\n", - " u'text': u'gold'}],\n", - " u'id': u'56bebc383aeaaa14008c931f',\n", - " u'question': u'What color was featured in promotions related to Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 290,\n", - " u'text': u'week 7'},\n", - " {u'answer_start': 72,\n", - " u'text': u'2015'},\n", - " {u'answer_start': 290,\n", - " u'text': u'week 7,'}],\n", - " u'id': u'56bebc383aeaaa14008c9320',\n", - " u'question': u'When did sideline jackets and hats start to feature gold-trimmed logos?'},\n", - " {u'answers': [{u'answer_start': 232,\n", - " u'text': u'50'},\n", - " {u'answer_start': 232,\n", - " u'text': u'50-yard line'},\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " {u'answer_start': 232,\n", - " u'text': u'50'}],\n", - " u'id': u'56bebc383aeaaa14008c9321',\n", - " u'question': u'What yard marker on the field was painted gold?'},\n", - " {u'answers': [{u'answer_start': 8,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 8,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 267,\n", - " u'text': u'gold'}],\n", - " u'id': u'56bf52483aeaaa14008c95c7',\n", - " u'question': u\"What color was used to create the 50-yard line in Levi's Stadium for the season leading up to the Super Bowl 50 game?\"},\n", - " {u'answers': [{u'answer_start': 8,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 8,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 357,\n", - " u'text': u'Gold'}],\n", - " u'id': u'56bf52483aeaaa14008c95c8',\n", - " u'question': u'What color were the footballs handed out to a variety of high schools, leading up to Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 105,\n", - " u'text': u'Golden Super Bowl'},\n", - " {u'answer_start': 100,\n", - " u'text': u'the \"Golden Super Bowl\"'},\n", - " {u'answer_start': 105,\n", - " u'text': u'Golden Super Bowl'}],\n", - " u'id': u'56d70e570d65d2140019833c',\n", - " u'question': u'Because of the gold theme, what was Super Bowl 50 known as?'},\n", - " {u'answers': [{u'answer_start': 357,\n", - " u'text': u'Gold footballs'},\n", - " {u'answer_start': 357,\n", - " u'text': u'Gold footballs'},\n", - " {u'answer_start': 357,\n", - " u'text': u'Gold footballs'}],\n", - " u'id': u'56d70e570d65d2140019833e',\n", - " u'question': u'What was given to high schools where former students went on to play or coach in a Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 228,\n", - " u'text': u'the 50-yard line'},\n", - " {u'answer_start': 228,\n", - " u'text': u'the 50'},\n", - " {u'answer_start': 232,\n", - " u'text': u'50'}],\n", - " u'id': u'56d70e570d65d2140019833f',\n", - " u'question': u'What yard line was a gold color for all teams during the season?'},\n", - " {u'answers': [{u'answer_start': 8,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 8,\n", - " u'text': u'gold'},\n", - " {u'answer_start': 8,\n", - " u'text': u'gold'}],\n", - " u'id': u'56d9b01fdc89441400fdb6d4',\n", - " u'question': u'Since this was the 50th one, what was the theme color for the Super Bowl?'}]},\n", - " {u'context': u'The annual NFL Experience was held at the Moscone Center in San Francisco. In addition, \"Super Bowl City\" opened on January 30 at Justin Herman Plaza on The Embarcadero, featuring games and activities that will highlight the Bay Area\\'s technology, culinary creations, and cultural diversity. More than 1 million people are expected to attend the festivities in San Francisco during Super Bowl Week. San Francisco mayor Ed Lee said of the highly visible homeless presence in this area \"they are going to have to leave\". San Francisco city supervisor Jane Kim unsuccessfully lobbied for the NFL to reimburse San Francisco for city services in the amount of $5 million.',\n", - " u'qas': [{u'answers': [{u'answer_start': 42,\n", - " u'text': u'Moscone Center'},\n", - " {u'answer_start': 38,\n", - " u'text': u'the Moscone Center'},\n", - " {u'answer_start': 42,\n", - " u'text': u'Moscone Center'}],\n", - " u'id': u'56bebcbe3aeaaa14008c9325',\n", - " u'question': u'In what venue did the NFL Experience take place?'},\n", - " {u'answers': [{u'answer_start': 60,\n", - " u'text': u'San Francisco'},\n", - " {u'answer_start': 60,\n", - " u'text': u'San Francisco'},\n", - " {u'answer_start': 60,\n", - " u'text': u'San Francisco'}],\n", - " u'id': u'56bebcbe3aeaaa14008c9326',\n", - " u'question': u'In what city is the Moscone Center located?'},\n", - " {u'answers': [{u'answer_start': 419,\n", - " u'text': u'Ed Lee'},\n", - " {u'answer_start': 419,\n", - " u'text': u'Ed Lee'},\n", - " {u'answer_start': 422,\n", - " u'text': u'Lee'}],\n", - " u'id': u'56bebcbe3aeaaa14008c9327',\n", - " u'question': u'Who is the mayor of San Francisco?'},\n", - " {u'answers': [{u'answer_start': 549,\n", - " u'text': u'Jane Kim'},\n", - " {u'answer_start': 549,\n", - " u'text': u'Jane Kim'},\n", - " {u'answer_start': 554,\n", - " u'text': u'Kim'}],\n", - " u'id': u'56bebcbe3aeaaa14008c9328',\n", - " u'question': u'Who is the city supervisor of San Francisco?'},\n", - " {u'answers': [{u'answer_start': 116,\n", - " u'text': u'January 30'},\n", - " {u'answer_start': 116,\n", - " u'text': u'January 30'},\n", - " {u'answer_start': 116,\n", - " u'text': u'January 30'}],\n", - " u'id': u'56bebcbe3aeaaa14008c9329',\n", - " u'question': u'When was the opening of \"Super Bowl City\"?'},\n", - " {u'answers': [{u'answer_start': 302,\n", - " u'text': u'1 million'},\n", - " {u'answer_start': 292,\n", - " u'text': u'More than 1 million'},\n", - " {u'answer_start': 302,\n", - " u'text': u'1 million'}],\n", - " u'id': u'56bf53e73aeaaa14008c95cb',\n", - " u'question': u'How many people were estimated to go to the different game-themed events during the week prior to Super Bowl 50 in the city it was held in?'},\n", - " {u'answers': [{u'answer_start': 419,\n", - " u'text': u'Ed Lee'},\n", - " {u'answer_start': 419,\n", - " u'text': u'Ed Lee'},\n", - " {u'answer_start': 422,\n", - " u'text': u'Lee'}],\n", - " u'id': u'56bf53e73aeaaa14008c95cc',\n", - " u'question': u'Who was the mayor of San Francisco during Super Bowl 50? '},\n", - " {u'answers': [{u'answer_start': 42,\n", - " u'text': u'Moscone Center'},\n", - " {u'answer_start': 38,\n", - " u'text': u'the Moscone Center'},\n", - " {u'answer_start': 42,\n", - " u'text': u'Moscone Center'}],\n", - " u'id': u'56bf53e73aeaaa14008c95cd',\n", - " u'question': u'Which building was the NFL Experience held at for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 89,\n", - " u'text': u'Super Bowl City'},\n", - " {u'answer_start': 89,\n", - " u'text': u'Super Bowl City'},\n", - " {u'answer_start': 89,\n", - " u'text': u'Super Bowl City'}],\n", - " u'id': u'56bf53e73aeaaa14008c95ce',\n", - " u'question': u'What was the name of the event at the The Embarcadero that was held prior to Super Bowl 50 to help show off some of the things that San Francisco has to offer?'},\n", - " {u'answers': [{u'answer_start': 42,\n", - " u'text': u'Moscone Center'},\n", - " {u'answer_start': 35,\n", - " u'text': u'at the Moscone Center'},\n", - " {u'answer_start': 42,\n", - " u'text': u'Moscone Center'}],\n", - " u'id': u'56d7145c0d65d2140019834c',\n", - " u'question': u'Where was the NFL Experience held for the 2015 season?'},\n", - " {u'answers': [{u'answer_start': 89,\n", - " u'text': u'Super Bowl City'},\n", - " {u'answer_start': 89,\n", - " u'text': u'Super Bowl City\" opened'},\n", - " {u'answer_start': 88,\n", - " u'text': u'\"Super Bowl City'}],\n", - " u'id': u'56d7145c0d65d2140019834d',\n", - " u'question': u'What opened on January 30th at the Justin Herman plaza?'},\n", - " {u'answers': [{u'answer_start': 419,\n", - " u'text': u'Ed Lee'},\n", - " {u'answer_start': 419,\n", - " u'text': u'Ed Lee'},\n", - " {u'answer_start': 422,\n", - " u'text': u'Lee'}],\n", - " u'id': u'56d7145c0d65d2140019834f',\n", - " u'question': u'Who is the mayor of San Francisco?'},\n", - " {u'answers': [{u'answer_start': 89,\n", - " u'text': u'Super Bowl City'},\n", - " {u'answer_start': 88,\n", - " u'text': u'\"Super Bowl City\"'},\n", - " {u'answer_start': 89,\n", - " u'text': u'Super Bowl City'}],\n", - " u'id': u'56d9b389dc89441400fdb6ec',\n", - " u'question': u'What football associated thing started at Justin Herman Plaza in January?'},\n", - " {u'answers': [{u'answer_start': 292,\n", - " u'text': u'More than 1 million'},\n", - " {u'answer_start': 292,\n", - " u'text': u'More than 1 million'},\n", - " {u'answer_start': 302,\n", - " u'text': u'1 million'}],\n", - " u'id': u'56d9b389dc89441400fdb6ed',\n", - " u'question': u'How many people are likely to visit Justin Herman Plaza during the week of the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 413,\n", - " u'text': u'mayor Ed Lee'},\n", - " {u'answer_start': 419,\n", - " u'text': u'Ed Lee'},\n", - " {u'answer_start': 413,\n", - " u'text': u'mayor Ed Lee'}],\n", - " u'id': u'56d9b389dc89441400fdb6ee',\n", - " u'question': u'Who said the homeless in the area would have to leave?'},\n", - " {u'answers': [{u'answer_start': 655,\n", - " u'text': u'$5 million.'},\n", - " {u'answer_start': 125,\n", - " u'text': u'0'},\n", - " {u'answer_start': 655,\n", - " u'text': u'$5 million'}],\n", - " u'id': u'56d9b389dc89441400fdb6ef',\n", - " u'question': u'How much did San Francisco get for providing services for the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'The annual NFL Experience'},\n", - " {u'answer_start': 0,\n", - " u'text': u'The annual NFL Experience'},\n", - " {u'answer_start': 11,\n", - " u'text': u'NFL Experience'}],\n", - " u'id': u'56d9b389dc89441400fdb6f0',\n", - " u'question': u'What did the Moscone Center host?'}]},\n", - " {u'context': u'In addition, there are $2 million worth of other ancillary events, including a week-long event at the Santa Clara Convention Center, a beer, wine and food festival at Bellomy Field at Santa Clara University, and a pep rally. A professional fundraiser will aid in finding business sponsors and individual donors, but still may need the city council to help fund the event. Additional funding will be provided by the city council, which has announced plans to set aside seed funding for the event.',\n", - " u'qas': [{u'answers': [{u'answer_start': 184,\n", - " u'text': u'Santa Clara University'},\n", - " {u'answer_start': 184,\n", - " u'text': u'Santa Clara University'},\n", - " {u'answer_start': 184,\n", - " u'text': u'Santa Clara University'}],\n", - " u'id': u'56bebd713aeaaa14008c932f',\n", - " u'question': u'At what university is Bellomy Field located?'},\n", - " {u'answers': [{u'answer_start': 23,\n", - " u'text': u'$2 million'},\n", - " {u'answer_start': 23,\n", - " u'text': u'$2 million'},\n", - " {u'answer_start': 23,\n", - " u'text': u'$2 million'}],\n", - " u'id': u'56bebd713aeaaa14008c9330',\n", - " u'question': u'How much money is being spent on other Super Bowl-related events?'},\n", - " {u'answers': [{u'answer_start': 77,\n", - " u'text': u'a week'},\n", - " {u'answer_start': 77,\n", - " u'text': u'a week'},\n", - " {u'answer_start': 79,\n", - " u'text': u'week'}],\n", - " u'id': u'56bebd713aeaaa14008c9331',\n", - " u'question': u'How long will the event at Santa Clara Convention Center last?'},\n", - " {u'answers': [{u'answer_start': 23,\n", - " u'text': u'$2 million'},\n", - " {u'answer_start': 23,\n", - " u'text': u'$2 million'},\n", - " {u'answer_start': 23,\n", - " u'text': u'$2 million'}],\n", - " u'id': u'56bf555e3aeaaa14008c95d3',\n", - " u'question': u'How much money was spent on other festivities in the Bay area to help celebrate the coming Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 214,\n", - " u'text': u'pep rally'},\n", - " {u'answer_start': 214,\n", - " u'text': u'pep'},\n", - " {u'answer_start': 214,\n", - " u'text': u'pep'}],\n", - " u'id': u'56bf555e3aeaaa14008c95d4',\n", - " u'question': u'What type of rally was held?'},\n", - " {u'answers': [{u'answer_start': 335,\n", - " u'text': u'city council'},\n", - " {u'answer_start': 411,\n", - " u'text': u'the city council'},\n", - " {u'answer_start': 415,\n", - " u'text': u'city council'}],\n", - " u'id': u'56bf555e3aeaaa14008c95d5',\n", - " u'question': u'Which government entity helped to pay for the festivities, beyond businesses and individuals?'},\n", - " {u'answers': [{u'answer_start': 167,\n", - " u'text': u'Bellomy Field'},\n", - " {u'answer_start': 167,\n", - " u'text': u'Bellomy Field'},\n", - " {u'answer_start': 102,\n", - " u'text': u'Santa Clara Convention Center'}],\n", - " u'id': u'56d714cd0d65d21400198356',\n", - " u'question': u'Where was a beer, wine and food festival held at prior to the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 225,\n", - " u'text': u'A professional fundraiser'},\n", - " {u'answer_start': 225,\n", - " u'text': u'A professional fundraiser'},\n", - " {u'answer_start': 227,\n", - " u'text': u'professional fundraiser'}],\n", - " u'id': u'56d714cd0d65d21400198358',\n", - " u'question': u'Who helped find sponsors and donors to help with the cost?'},\n", - " {u'answers': [{u'answer_start': 335,\n", - " u'text': u'city council'},\n", - " {u'answer_start': 411,\n", - " u'text': u'the city council'},\n", - " {u'answer_start': 415,\n", - " u'text': u'city council'}],\n", - " u'id': u'56d714cd0d65d21400198359',\n", - " u'question': u'Who set aside seed funding for the week-long event leading up to the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 23,\n", - " u'text': u'$2 million'},\n", - " {u'answer_start': 23,\n", - " u'text': u'$2 million'},\n", - " {u'answer_start': 23,\n", - " u'text': u'$2 million'}],\n", - " u'id': u'56d9b43edc89441400fdb700',\n", - " u'question': u'What was the cost of the other Super Bowl events in the San Francisco area?'},\n", - " {u'answers': [{u'answer_start': 335,\n", - " u'text': u'city council'},\n", - " {u'answer_start': 411,\n", - " u'text': u'the city council'},\n", - " {u'answer_start': 335,\n", - " u'text': u'city council'}],\n", - " u'id': u'56d9b43edc89441400fdb701',\n", - " u'question': u'Who may have been called upon to fund the festival associated with the Super Bowl in Santa Clara?'}]},\n", - " {u'context': u\"The game's media day, which was typically held on the Tuesday afternoon prior to the game, was moved to the Monday evening and re-branded as Super Bowl Opening Night. The event was held on February 1, 2016 at SAP Center in San Jose. Alongside the traditional media availabilities, the event featured an opening ceremony with player introductions on a replica of the Golden Gate Bridge.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 108,\n", - " u'text': u'Monday'},\n", - " {u'answer_start': 189,\n", - " u'text': u'February 1, 2016'},\n", - " {u'answer_start': 108,\n", - " u'text': u'Monday'}],\n", - " u'id': u'56bebde53aeaaa14008c9335',\n", - " u'question': u'On what day was Super Bowl Opening Night held?'},\n", - " {u'answers': [{u'answer_start': 54,\n", - " u'text': u'Tuesday'},\n", - " {u'answer_start': 54,\n", - " u'text': u'Tuesday afternoon prior to the game'},\n", - " {u'answer_start': 54,\n", - " u'text': u'Tuesday'}],\n", - " u'id': u'56bebde53aeaaa14008c9336',\n", - " u'question': u'On what day does the Super Bowl media day traditionally take place?'},\n", - " {u'answers': [{u'answer_start': 209,\n", - " u'text': u'SAP Center'},\n", - " {u'answer_start': 209,\n", - " u'text': u'SAP Center'},\n", - " {u'answer_start': 209,\n", - " u'text': u'SAP Center'}],\n", - " u'id': u'56bebde53aeaaa14008c9337',\n", - " u'question': u'What venue hosted Super Bowl Opening Night?'},\n", - " {u'answers': [{u'answer_start': 223,\n", - " u'text': u'San Jose'},\n", - " {u'answer_start': 223,\n", - " u'text': u'San Jose'},\n", - " {u'answer_start': 223,\n", - " u'text': u'San Jose.'}],\n", - " u'id': u'56bebde53aeaaa14008c9338',\n", - " u'question': u'In what city is SAP Center located?'},\n", - " {u'answers': [{u'answer_start': 362,\n", - " u'text': u'the Golden Gate Bridge'},\n", - " {u'answer_start': 362,\n", - " u'text': u'the Golden Gate Bridge'},\n", - " {u'answer_start': 366,\n", - " u'text': u'Golden Gate Bridge.'}],\n", - " u'id': u'56bebde53aeaaa14008c9339',\n", - " u'question': u'A replica of what landmark was present at Super Bowl Opening Night?'},\n", - " {u'answers': [{u'answer_start': 54,\n", - " u'text': u'Tuesday'},\n", - " {u'answer_start': 54,\n", - " u'text': u'Tuesday afternoon prior to the game'},\n", - " {u'answer_start': 54,\n", - " u'text': u'Tuesday'}],\n", - " u'id': u'56bf57043aeaaa14008c95d9',\n", - " u'question': u'As a norm, what day of the week is the traditional Media Day held prior to a Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 108,\n", - " u'text': u'Monday'},\n", - " {u'answer_start': 108,\n", - " u'text': u'Monday'},\n", - " {u'answer_start': 108,\n", - " u'text': u'Monday'}],\n", - " u'id': u'56bf57043aeaaa14008c95da',\n", - " u'question': u'What day of the week was Media Day held on for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 141,\n", - " u'text': u'Super Bowl Opening Night'},\n", - " {u'answer_start': 141,\n", - " u'text': u'Super Bowl Opening Night'},\n", - " {u'answer_start': 141,\n", - " u'text': u'Super Bowl Opening Night.'}],\n", - " u'id': u'56bf57043aeaaa14008c95db',\n", - " u'question': u'What was the name of the Media Day event for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 209,\n", - " u'text': u'SAP Center'},\n", - " {u'answer_start': 209,\n", - " u'text': u'SAP Center'},\n", - " {u'answer_start': 209,\n", - " u'text': u'SAP Center'}],\n", - " u'id': u'56bf57043aeaaa14008c95dc',\n", - " u'question': u'What is the name of the property where the media event was held for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 223,\n", - " u'text': u'San Jose'},\n", - " {u'answer_start': 223,\n", - " u'text': u'San Jose'},\n", - " {u'answer_start': 223,\n", - " u'text': u'San Jose.'}],\n", - " u'id': u'56bf57043aeaaa14008c95dd',\n", - " u'question': u'What city was the media event held for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 50,\n", - " u'text': u'the Tuesday afternoon prior to the game'},\n", - " {u'answer_start': 54,\n", - " u'text': u'Tuesday afternoon prior to the game'},\n", - " {u'answer_start': 54,\n", - " u'text': u'Tuesday'}],\n", - " u'id': u'56d71bd80d65d2140019835e',\n", - " u'question': u\"When is the game's media day usually held?\"},\n", - " {u'answers': [{u'answer_start': 141,\n", - " u'text': u'Super Bowl Opening Night.'},\n", - " {u'answer_start': 141,\n", - " u'text': u'Super Bowl Opening Night'},\n", - " {u'answer_start': 141,\n", - " u'text': u'Super Bowl Opening Night.'}],\n", - " u'id': u'56d71bd80d65d21400198360',\n", - " u'question': u'What new name was given to the media day?'},\n", - " {u'answers': [{u'answer_start': 209,\n", - " u'text': u'SAP Center in San Jose.'},\n", - " {u'answer_start': 206,\n", - " u'text': u'at SAP Center in San Jose'},\n", - " {u'answer_start': 223,\n", - " u'text': u'San Jose'}],\n", - " u'id': u'56d71bd80d65d21400198361',\n", - " u'question': u'Where was media day for Super Bowl 50 held?'},\n", - " {u'answers': [{u'answer_start': 362,\n", - " u'text': u'the Golden Gate Bridge.'},\n", - " {u'answer_start': 362,\n", - " u'text': u'the Golden Gate Bridge'},\n", - " {u'answer_start': 366,\n", - " u'text': u'Golden Gate Bridge.'}],\n", - " u'id': u'56d71bd80d65d21400198362',\n", - " u'question': u'What replica was used for player introductions?'},\n", - " {u'answers': [{u'answer_start': 108,\n", - " u'text': u'Monday'},\n", - " {u'answer_start': 108,\n", - " u'text': u'Monday'},\n", - " {u'answer_start': 108,\n", - " u'text': u'Monday'}],\n", - " u'id': u'56d9b4ebdc89441400fdb708',\n", - " u'question': u'What day was Super Bowl 50 media day switched to?'},\n", - " {u'answers': [{u'answer_start': 141,\n", - " u'text': u'Super Bowl Opening Night.'},\n", - " {u'answer_start': 141,\n", - " u'text': u'Super Bowl Opening Night'},\n", - " {u'answer_start': 141,\n", - " u'text': u'Super Bowl Opening Night'}],\n", - " u'id': u'56d9b4ebdc89441400fdb709',\n", - " u'question': u'What was media day called for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 209,\n", - " u'text': u'SAP Center in San Jose.'},\n", - " {u'answer_start': 209,\n", - " u'text': u'SAP Center'},\n", - " {u'answer_start': 209,\n", - " u'text': u'SAP Center'}],\n", - " u'id': u'56d9b4ebdc89441400fdb70a',\n", - " u'question': u'Where was the new media day event for Super Bowl 50 held?'},\n", - " {u'answers': [{u'answer_start': 366,\n", - " u'text': u'Golden Gate Bridge.'},\n", - " {u'answer_start': 362,\n", - " u'text': u'the Golden Gate'},\n", - " {u'answer_start': 366,\n", - " u'text': u'Golden Gate Bridge.'}],\n", - " u'id': u'56d9b4ebdc89441400fdb70b',\n", - " u'question': u'What bridge had a model of it at the Super Bowl Opening Night?'},\n", - " {u'answers': [{u'answer_start': 189,\n", - " u'text': u'February 1, 2016'},\n", - " {u'answer_start': 189,\n", - " u'text': u'February 1, 2016'},\n", - " {u'answer_start': 189,\n", - " u'text': u'February 1, 2016'}],\n", - " u'id': u'56d9b4ebdc89441400fdb70c',\n", - " u'question': u'What date was Super Bowl Opening Night held?'}]},\n", - " {u'context': u'For the first time, the Super Bowl 50 Host Committee and the NFL have openly sought disabled veteran and lesbian, gay, bisexual and transgender-owned businesses in Business Connect, the Super Bowl program that provides local companies with contracting opportunities in and around the Super Bowl. The host committee has already raised over $40 million through sponsors including Apple, Google, Yahoo!, Intel, Gap, Chevron, and Dignity Health.',\n", - " u'qas': [{u'answers': [{u'answer_start': 164,\n", - " u'text': u'Business Connect'},\n", - " {u'answer_start': 164,\n", - " u'text': u'Business Connect'},\n", - " {u'answer_start': 164,\n", - " u'text': u'Business Connect'}],\n", - " u'id': u'56bebe873aeaaa14008c933f',\n", - " u'question': u'What is the name of the program that provides contracting work to local companies?'},\n", - " {u'answers': [{u'answer_start': 339,\n", - " u'text': u'$40 million'},\n", - " {u'answer_start': 334,\n", - " u'text': u'over $40 million'},\n", - " {u'answer_start': 339,\n", - " u'text': u'$40 million'}],\n", - " u'id': u'56bebe873aeaaa14008c9340',\n", - " u'question': u'How much money has been raised by the host committee?'},\n", - " {u'answers': [{u'answer_start': 426,\n", - " u'text': u'Dignity Health'},\n", - " {u'answer_start': 426,\n", - " u'text': u'Dignity Health'},\n", - " {u'answer_start': 426,\n", - " u'text': u'Dignity'}],\n", - " u'id': u'56bebe873aeaaa14008c9341',\n", - " u'question': u'What health company was a Super Bowl sponsor?'},\n", - " {u'answers': [{u'answer_start': 408,\n", - " u'text': u'Gap'},\n", - " {u'answer_start': 408,\n", - " u'text': u'Gap'},\n", - " {u'answer_start': 408,\n", - " u'text': u'Gap'}],\n", - " u'id': u'56bebe873aeaaa14008c9342',\n", - " u'question': u'What clothing company was a Super Bowl sponsor?'},\n", - " {u'answers': [{u'answer_start': 413,\n", - " u'text': u'Chevron'},\n", - " {u'answer_start': 413,\n", - " u'text': u'Chevron'},\n", - " {u'answer_start': 413,\n", - " u'text': u'Chevron'}],\n", - " u'id': u'56bebe873aeaaa14008c9343',\n", - " u'question': u'What petroleum company was a Super Bowl sponsor?'},\n", - " {u'answers': [{u'answer_start': 24,\n", - " u'text': u'Super Bowl 50 Host Committee'},\n", - " {u'answer_start': 20,\n", - " u'text': u'the Super Bowl 50 Host Committee'},\n", - " {u'answer_start': 24,\n", - " u'text': u'Super Bowl 50 Host Committee'}],\n", - " u'id': u'56bf59523aeaaa14008c95e3',\n", - " u'question': u'Which committee made an effort to invite companies owned by people, who are transgender or gay, to take part in contract business associated with Super Bowl 50.'},\n", - " {u'answers': [{u'answer_start': 334,\n", - " u'text': u'over $40 million'},\n", - " {u'answer_start': 334,\n", - " u'text': u'over $40 million'},\n", - " {u'answer_start': 334,\n", - " u'text': u'over $40 million'}],\n", - " u'id': u'56bf59523aeaaa14008c95e4',\n", - " u'question': u'How much money did the committee raise?'},\n", - " {u'answers': [{u'answer_start': 359,\n", - " u'text': u'sponsors'},\n", - " {u'answer_start': 351,\n", - " u'text': u'through sponsors'},\n", - " {u'answer_start': 359,\n", - " u'text': u'sponsors'}],\n", - " u'id': u'56bf59523aeaaa14008c95e5',\n", - " u'question': u'How did the committee raise the money?'},\n", - " {u'answers': [{u'answer_start': 164,\n", - " u'text': u'Business Connect'},\n", - " {u'answer_start': 164,\n", - " u'text': u'Business Connect'},\n", - " {u'answer_start': 164,\n", - " u'text': u'Business Connect'}],\n", - " u'id': u'56d71cb60d65d21400198368',\n", - " u'question': u'What is the Super Bowl program called that gives local companies business opportunities for the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 164,\n", - " u'text': u'Business Connect'},\n", - " {u'answer_start': 164,\n", - " u'text': u'Business Connect'},\n", - " {u'answer_start': 164,\n", - " u'text': u'Business Connect'}],\n", - " u'id': u'56d9b5ffdc89441400fdb720',\n", - " u'question': u'What is the name of the service that gets local businesses contract chances with the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 334,\n", - " u'text': u'over $40 million'},\n", - " {u'answer_start': 334,\n", - " u'text': u'over $40 million'},\n", - " {u'answer_start': 339,\n", - " u'text': u'$40 million'}],\n", - " u'id': u'56d9b5ffdc89441400fdb721',\n", - " u'question': u'According to the text, how much money did the Host Committee raise?'}]},\n", - " {u'context': u'The Super Bowl 50 Host Committee has vowed to be \"the most giving Super Bowl ever\", and will dedicate 25 percent of all money it raises for philanthropic causes in the Bay Area. The committee created the 50 fund as its philanthropic initiative and focuses on providing grants to aid with youth development, community investment and sustainable environments.',\n", - " u'qas': [{u'answers': [{u'answer_start': 102,\n", - " u'text': u'25'},\n", - " {u'answer_start': 102,\n", - " u'text': u'25 percent'},\n", - " {u'answer_start': 102,\n", - " u'text': u'25'}],\n", - " u'id': u'56bebec43aeaaa14008c9349',\n", - " u'question': u'What percentage of money raised by the Host Committee will be used for charitable causes?'},\n", - " {u'answers': [{u'answer_start': 200,\n", - " u'text': u'the 50 fund'},\n", - " {u'answer_start': 200,\n", - " u'text': u'the 50 fund'},\n", - " {u'answer_start': 204,\n", - " u'text': u'50 fund'}],\n", - " u'id': u'56bebec43aeaaa14008c934a',\n", - " u'question': u\"What is the name of the Host Committee's charitable initiative?\"},\n", - " {u'answers': [{u'answer_start': 102,\n", - " u'text': u'25 percent'},\n", - " {u'answer_start': 102,\n", - " u'text': u'25 percent'},\n", - " {u'answer_start': 102,\n", - " u'text': u'25'}],\n", - " u'id': u'56bf5abc3aeaaa14008c95e9',\n", - " u'question': u'What percentage of funds were given as charitable funds to causes in and around San Francisco?'},\n", - " {u'answers': [{u'answer_start': 204,\n", - " u'text': u'50 fund'},\n", - " {u'answer_start': 200,\n", - " u'text': u'the 50 fund'},\n", - " {u'answer_start': 204,\n", - " u'text': u'50 fund'}],\n", - " u'id': u'56bf5abc3aeaaa14008c95ea',\n", - " u'question': u'What was the name of the fund setup to help with investing in the community? '},\n", - " {u'answers': [{u'answer_start': 50,\n", - " u'text': u'the most giving Super Bowl ever'},\n", - " {u'answer_start': 59,\n", - " u'text': u'giving Super Bowl'},\n", - " {u'answer_start': 59,\n", - " u'text': u'giving'}],\n", - " u'id': u'56d71d150d65d2140019836e',\n", - " u'question': u'The Super Bowl 50 Host Committee said it would be the most what ever?'},\n", - " {u'answers': [{u'answer_start': 102,\n", - " u'text': u'25 percent'},\n", - " {u'answer_start': 102,\n", - " u'text': u'25 percent of all money'},\n", - " {u'answer_start': 102,\n", - " u'text': u'25'}],\n", - " u'id': u'56d71d150d65d2140019836f',\n", - " u'question': u'How many will the host committee dedicate to local charities?'},\n", - " {u'answers': [{u'answer_start': 200,\n", - " u'text': u'the 50 fund'},\n", - " {u'answer_start': 200,\n", - " u'text': u'the 50 fund'},\n", - " {u'answer_start': 204,\n", - " u'text': u'50 fund'}],\n", - " u'id': u'56d71d150d65d21400198370',\n", - " u'question': u'What is the name of the fund that focuses on youth, community and sustainable environments?'},\n", - " {u'answers': [{u'answer_start': 102,\n", - " u'text': u'25 percent'},\n", - " {u'answer_start': 102,\n", - " u'text': u'25 percent'},\n", - " {u'answer_start': 102,\n", - " u'text': u'25'}],\n", - " u'id': u'56d9b6a3dc89441400fdb72f',\n", - " u'question': u'What percentage of money raised was earmarked for causes in the San Francisco area?'},\n", - " {u'answers': [{u'answer_start': 204,\n", - " u'text': u'50 fund'},\n", - " {u'answer_start': 200,\n", - " u'text': u'the 50 fund'},\n", - " {u'answer_start': 204,\n", - " u'text': u'50 fund'}],\n", - " u'id': u'56d9b6a3dc89441400fdb730',\n", - " u'question': u'What was the name of the work to give grants for youth development and other things?'}]},\n", - " {u'context': u'In addition to the Vince Lombardi Trophy that all Super Bowl champions receive, the winner of Super Bowl 50 will also receive a large, 18-karat gold-plated \"50\". Each digit will weigh 33 lb (15 kg) for a total of 66 lb (30 kg). Like the Lombardi Trophy, the \"50\" will be designed by Tiffany & Co.',\n", - " u'qas': [{u'answers': [{u'answer_start': 19,\n", - " u'text': u'Vince Lombardi'},\n", - " {u'answer_start': 19,\n", - " u'text': u'Vince Lombardi'},\n", - " {u'answer_start': 25,\n", - " u'text': u'Lombardi'}],\n", - " u'id': u'56bec0353aeaaa14008c934d',\n", - " u'question': u'Who is the trophy given to the Super Bowl champion named for?'},\n", - " {u'answers': [{u'answer_start': 135,\n", - " u'text': u'18'},\n", - " {u'answer_start': 135,\n", - " u'text': u'18'},\n", - " {u'answer_start': 135,\n", - " u'text': u'18'}],\n", - " u'id': u'56bec0353aeaaa14008c934e',\n", - " u'question': u'The \"50\" given to the Super Bowl winner is plated with how many karats of gold?'},\n", - " {u'answers': [{u'answer_start': 213,\n", - " u'text': u'66'},\n", - " {u'answer_start': 213,\n", - " u'text': u'66 lb'},\n", - " {u'answer_start': 184,\n", - " u'text': u'33'}],\n", - " u'id': u'56bec0353aeaaa14008c934f',\n", - " u'question': u'What will the \"50\" given to the Super Bowl winner weigh in pounds?'},\n", - " {u'answers': [{u'answer_start': 283,\n", - " u'text': u'Tiffany & Co.'},\n", - " {u'answer_start': 283,\n", - " u'text': u'Tiffany & Co'},\n", - " {u'answer_start': 283,\n", - " u'text': u'Tiffany & Co.'}],\n", - " u'id': u'56bec0353aeaaa14008c9350',\n", - " u'question': u'Who designed the \"50\"?'},\n", - " {u'answers': [{u'answer_start': 283,\n", - " u'text': u'Tiffany & Co.'},\n", - " {u'answer_start': 283,\n", - " u'text': u'Tiffany & Co'},\n", - " {u'answer_start': 283,\n", - " u'text': u'Tiffany & Co'}],\n", - " u'id': u'56bec0353aeaaa14008c9351',\n", - " u'question': u'Who designed the Vince Lombardi Trophy?'},\n", - " {u'answers': [{u'answer_start': 19,\n", - " u'text': u'Vince Lombardi Trophy'},\n", - " {u'answer_start': 15,\n", - " u'text': u'the Vince Lombardi Trophy'},\n", - " {u'answer_start': 19,\n", - " u'text': u'Vince Lombardi Trophy'}],\n", - " u'id': u'56bf5b933aeaaa14008c95ed',\n", - " u'question': u'What is the name of the trophy given to anyone who plays on the winning team in a Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 135,\n", - " u'text': u'18-karat gold-plated'},\n", - " {u'answer_start': 135,\n", - " u'text': u'18-karat gold'},\n", - " {u'answer_start': 144,\n", - " u'text': u'gold'}],\n", - " u'id': u'56bf5b933aeaaa14008c95ef',\n", - " u'question': u'What will the number \"50\" be plated with? '},\n", - " {u'answers': [{u'answer_start': 283,\n", - " u'text': u'Tiffany & Co'},\n", - " {u'answer_start': 283,\n", - " u'text': u'Tiffany & Co'},\n", - " {u'answer_start': 283,\n", - " u'text': u'Tiffany & Co'}],\n", - " u'id': u'56bf5b933aeaaa14008c95f1',\n", - " u'question': u'Who is the designer of the \"50?\"'},\n", - " {u'answers': [{u'answer_start': 15,\n", - " u'text': u'the Vince Lombardi Trophy'},\n", - " {u'answer_start': 15,\n", - " u'text': u'the Vince Lombardi Trophy'},\n", - " {u'answer_start': 19,\n", - " u'text': u'Vince Lombardi Trophy'}],\n", - " u'id': u'56d71d9d0d65d21400198374',\n", - " u'question': u'What is the name of the trophy that all Super Bowl winners receive?'},\n", - " {u'answers': [{u'answer_start': 283,\n", - " u'text': u'Tiffany & Co.'},\n", - " {u'answer_start': 283,\n", - " u'text': u'Tiffany & Co'},\n", - " {u'answer_start': 283,\n", - " u'text': u'Tiffany & Co.'}],\n", - " u'id': u'56d71d9d0d65d21400198377',\n", - " u'question': u'Who designs both the \"50\" as well as the Trophy?'}]},\n", - " {u'context': u'In the United States, the game was televised by CBS, as part of a cycle between the three main broadcast television partners of the NFL. The network\\'s lead broadcast team of Jim Nantz and Phil Simms called the contest, with Tracy Wolfson and Evan Washburn on the sidelines. CBS introduced new features during the telecast, including pylon cameras and microphones along with EyeVision 360\\u2014an array of 36 cameras along the upper deck that can be used to provide a 360-degree view of plays and \"bullet time\" effects. (An earlier version of EyeVision was last used in Super Bowl XXXV; for Super Bowl 50, the cameras were upgraded to 5K resolution.)',\n", - " u'qas': [{u'answers': [{u'answer_start': 48,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 48,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 48,\n", - " u'text': u'CBS'}],\n", - " u'id': u'56bec0dd3aeaaa14008c9357',\n", - " u'question': u'On what television station could an American viewer watch the game?'},\n", - " {u'answers': [{u'answer_start': 188,\n", - " u'text': u'Phil Simms'},\n", - " {u'answer_start': 188,\n", - " u'text': u'Phil Simms'},\n", - " {u'answer_start': 188,\n", - " u'text': u'Phil Simms'}],\n", - " u'id': u'56bec0dd3aeaaa14008c9358',\n", - " u'question': u\"Who, with Jim Nantz, was a member of the game's lead broadcast team?\"},\n", - " {u'answers': [{u'answer_start': 224,\n", - " u'text': u'Tracy Wolfson'},\n", - " {u'answer_start': 224,\n", - " u'text': u'Tracy Wolfson'},\n", - " {u'answer_start': 224,\n", - " u'text': u'Tracy Wolfson'}],\n", - " u'id': u'56bec0dd3aeaaa14008c9359',\n", - " u'question': u'Who shared sideline duties with Evan Washburn?'},\n", - " {u'answers': [{u'answer_start': 384,\n", - " u'text': u'36'},\n", - " {u'answer_start': 400,\n", - " u'text': u'36'},\n", - " {u'answer_start': 400,\n", - " u'text': u'36'}],\n", - " u'id': u'56bec0dd3aeaaa14008c935a',\n", - " u'question': u'How many cameras did EyeVision 360 utilize?'},\n", - " {u'answers': [{u'answer_start': 629,\n", - " u'text': u'5K'},\n", - " {u'answer_start': 629,\n", - " u'text': u'5K'},\n", - " {u'answer_start': 629,\n", - " u'text': u'5K'}],\n", - " u'id': u'56bec0dd3aeaaa14008c935b',\n", - " u'question': u'What was the resolution of the cameras used in the EyeVision 360 system?'},\n", - " {u'answers': [{u'answer_start': 84,\n", - " u'text': u'three'},\n", - " {u'answer_start': 84,\n", - " u'text': u'three'},\n", - " {u'answer_start': 84,\n", - " u'text': u'three'}],\n", - " u'id': u'56bf5e7b3aeaaa14008c95f7',\n", - " u'question': u'How many different TV networks are considered to be the principal partners, when it comes to broadcasting NFL events?'},\n", - " {u'answers': [{u'answer_start': 263,\n", - " u'text': u'sidelines'},\n", - " {u'answer_start': 259,\n", - " u'text': u'the sidelines'},\n", - " {u'answer_start': 263,\n", - " u'text': u'sidelines'}],\n", - " u'id': u'56bf5e7b3aeaaa14008c95f8',\n", - " u'question': u'Where was Tracy Wolfson announcing from during the Super Bowl 50 game?'},\n", - " {u'answers': [{u'answer_start': 462,\n", - " u'text': u'360-degree'},\n", - " {u'answer_start': 384,\n", - " u'text': u'360'},\n", - " {u'answer_start': 462,\n", - " u'text': u'360'}],\n", - " u'id': u'56bf5e7b3aeaaa14008c95f9',\n", - " u'question': u'New cameras were introduced for the game, what degree of view do they provide viewers?'},\n", - " {u'answers': [{u'answer_start': 629,\n", - " u'text': u'5K resolution'},\n", - " {u'answer_start': 629,\n", - " u'text': u'5K'},\n", - " {u'answer_start': 629,\n", - " u'text': u'5K'}],\n", - " u'id': u'56bf5e7b3aeaaa14008c95fa',\n", - " u'question': u'What resolution were the cameras increased to for the game?'},\n", - " {u'answers': [{u'answer_start': 263,\n", - " u'text': u'sidelines'},\n", - " {u'answer_start': 259,\n", - " u'text': u'the sidelines'},\n", - " {u'answer_start': 263,\n", - " u'text': u'sidelines'}],\n", - " u'id': u'56bf5e7b3aeaaa14008c95fb',\n", - " u'question': u'Where was Evan Washburn located while announcing during the game?'},\n", - " {u'answers': [{u'answer_start': 48,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 48,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 48,\n", - " u'text': u'CBS'}],\n", - " u'id': u'56d71ed50d65d2140019837c',\n", - " u'question': u'Which television network aired the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 84,\n", - " u'text': u'three'},\n", - " {u'answer_start': 84,\n", - " u'text': u'three'},\n", - " {u'answer_start': 84,\n", - " u'text': u'three'}],\n", - " u'id': u'56d71ed50d65d2140019837d',\n", - " u'question': u'How many broadcast television partners does the NFL have?'},\n", - " {u'answers': [{u'answer_start': 48,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 48,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 48,\n", - " u'text': u'CBS'}],\n", - " u'id': u'56d9b7dcdc89441400fdb740',\n", - " u'question': u'What network aired Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 174,\n", - " u'text': u'Jim Nantz and Phil Simms'},\n", - " {u'answer_start': 174,\n", - " u'text': u'Jim Nantz and Phil Simms'},\n", - " {u'answer_start': 174,\n", - " u'text': u'Jim Nantz and Phil Simms'}],\n", - " u'id': u'56d9b7dcdc89441400fdb741',\n", - " u'question': u'Who were the announcers of Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 224,\n", - " u'text': u'Tracy Wolfson and Evan Washburn'},\n", - " {u'answer_start': 224,\n", - " u'text': u'Tracy Wolfson and Evan Washburn'},\n", - " {u'answer_start': 224,\n", - " u'text': u'Tracy Wolfson and Evan Washburn'}],\n", - " u'id': u'56d9b7dcdc89441400fdb742',\n", - " u'question': u'Who were the Super Bowl 50 sideline announcers?'},\n", - " {u'answers': [{u'answer_start': 629,\n", - " u'text': u'5K'},\n", - " {u'answer_start': 629,\n", - " u'text': u'5K'},\n", - " {u'answer_start': 629,\n", - " u'text': u'5K'}],\n", - " u'id': u'56d9b7dcdc89441400fdb743',\n", - " u'question': u'What resolution were the Eyevision cameras changed to?'},\n", - " {u'answers': [{u'answer_start': 339,\n", - " u'text': u'cameras'},\n", - " {u'answer_start': 339,\n", - " u'text': u'cameras'},\n", - " {u'answer_start': 339,\n", - " u'text': u'cameras'}],\n", - " u'id': u'56d9b7dcdc89441400fdb744',\n", - " u'question': u'What was put on pylons for Super Bowl 50?'}]},\n", - " {u'context': u\"On December 28, 2015, ESPN Deportes announced that they had reached an agreement with CBS and the NFL to be the exclusive Spanish-language broadcaster of the game, marking the third dedicated Spanish-language broadcast of the Super Bowl. Unlike NBC and Fox, CBS does not have a Spanish-language outlet of its own that could broadcast the game (though per league policy, a separate Spanish play-by-play call was carried on CBS's second audio program channel for over-the-air viewers). The game was called by ESPN Deportes' Monday Night Football commentary crew of Alvaro Martin and Raul Allegre, and sideline reporter John Sutcliffe. ESPN Deportes broadcast pre-game and post-game coverage, while Martin, Allegre, and Sutcliffe contributed English-language reports for ESPN's SportsCenter and Mike & Mike.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 22,\n", - " u'text': u'ESPN Deportes'},\n", - " {u'answer_start': 22,\n", - " u'text': u'ESPN Deportes'},\n", - " {u'answer_start': 22,\n", - " u'text': u'ESPN Deportes'}],\n", - " u'id': u'56bec1823aeaaa14008c9361',\n", - " u'question': u'Which network broadcast the game in Spanish?'},\n", - " {u'answers': [{u'answer_start': 617,\n", - " u'text': u'John Sutcliffe'},\n", - " {u'answer_start': 617,\n", - " u'text': u'John Sutcliffe'},\n", - " {u'answer_start': 622,\n", - " u'text': u'Sutcliffe'}],\n", - " u'id': u'56bec1823aeaaa14008c9362',\n", - " u'question': u'Who reported on the sideline for ESPN Deportes?'},\n", - " {u'answers': [{u'answer_start': 563,\n", - " u'text': u'Alvaro Martin'},\n", - " {u'answer_start': 563,\n", - " u'text': u'Alvaro Martin'},\n", - " {u'answer_start': 563,\n", - " u'text': u'Alvaro Martin'}],\n", - " u'id': u'56bec1823aeaaa14008c9363',\n", - " u'question': u'Who provided Spanish-language commentary alongside Raul Allegre?'},\n", - " {u'answers': [{u'answer_start': 3,\n", - " u'text': u'December 28, 2015'},\n", - " {u'answer_start': 3,\n", - " u'text': u'December 28, 2015'},\n", - " {u'answer_start': 3,\n", - " u'text': u'December 28, 2015,'}],\n", - " u'id': u'56bec1823aeaaa14008c9364',\n", - " u'question': u'On what date did ESPN Deportes announce their deal with CBS and the NFL?'},\n", - " {u'answers': [{u'answer_start': 122,\n", - " u'text': u'Spanish'},\n", - " {u'answer_start': 122,\n", - " u'text': u'Spanish'},\n", - " {u'answer_start': 122,\n", - " u'text': u'Spanish'}],\n", - " u'id': u'56bec1823aeaaa14008c9365',\n", - " u'question': u'In what language did ESPN Deportes broadcast the game?'},\n", - " {u'answers': [{u'answer_start': 86,\n", - " u'text': u'CBS'},\n", - " {u'answer_start': 22,\n", - " u'text': u'ESPN Deportes'},\n", - " {u'answer_start': 22,\n", - " u'text': u'ESPN Deportes'}],\n", - " u'id': u'56bf6d343aeaaa14008c961b',\n", - " u'question': u'Which TV network broadcasted Super Bowl 50 in Spanish?'},\n", - " {u'answers': [{u'answer_start': 22,\n", - " u'text': u'ESPN Deportes'},\n", - " {u'answer_start': 22,\n", - " u'text': u'ESPN Deportes'},\n", - " {u'answer_start': 22,\n", - " u'text': u'ESPN Deportes'}],\n", - " u'id': u'56d71fc00d65d21400198386',\n", - " u'question': u'Who exclusively broadcast the game in Spanish after reaching an agreement with the NFL and CBS?'},\n", - " {u'answers': [{u'answer_start': 617,\n", - " u'text': u'John Sutcliffe.'},\n", - " {u'answer_start': 617,\n", - " u'text': u'John Sutcliffe'},\n", - " {u'answer_start': 622,\n", - " u'text': u'Sutcliffe'}],\n", - " u'id': u'56d71fc00d65d21400198389',\n", - " u'question': u'Who was the sideline reporter for ESPN Deportes?'},\n", - " {u'answers': [{u'answer_start': 22,\n", - " u'text': u'ESPN Deportes'},\n", - " {u'answer_start': 22,\n", - " u'text': u'ESPN Deportes'},\n", - " {u'answer_start': 22,\n", - " u'text': u'ESPN Deportes'}],\n", - " u'id': u'56d9ba95dc89441400fdb754',\n", - " u'question': u'What agency did the Spanish version of the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 563,\n", - " u'text': u'Alvaro Martin and Raul Allegre'},\n", - " {u'answer_start': 563,\n", - " u'text': u'Alvaro Martin and Raul Allegre'},\n", - " {u'answer_start': 563,\n", - " u'text': u'Alvaro Martin and Raul Allegre'}],\n", - " u'id': u'56d9ba95dc89441400fdb757',\n", - " u'question': u'Who were the ESPN Deportes commentators for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 617,\n", - " u'text': u'John Sutcliffe.'},\n", - " {u'answer_start': 617,\n", - " u'text': u'John Sutcliffe'},\n", - " {u'answer_start': 622,\n", - " u'text': u'Sutcliffe'}],\n", - " u'id': u'56d9ba95dc89441400fdb758',\n", - " u'question': u'Who was the ESPN Deportes sideline commentator for Super Bowl 50?'}]},\n", - " {u'context': u'CBS provided digital streams of the game via CBSSports.com, and the CBS Sports apps on tablets, Windows 10, Xbox One and other digital media players (such as Chromecast and Roku). Due to Verizon Communications exclusivity, streaming on smartphones was only provided to Verizon Wireless customers via the NFL Mobile service. The ESPN Deportes Spanish broadcast was made available through WatchESPN.',\n", - " u'qas': [{u'answers': [{u'answer_start': 304,\n", - " u'text': u'NFL Mobile'},\n", - " {u'answer_start': 304,\n", - " u'text': u'NFL Mobile'},\n", - " {u'answer_start': 304,\n", - " u'text': u'NFL Mobile service.'}],\n", - " u'id': u'56bec2013aeaaa14008c9371',\n", - " u'question': u'What service did Verizon customers need to use to stream the game on their smartphones?'},\n", - " {u'answers': [{u'answer_start': 387,\n", - " u'text': u'WatchESPN'},\n", - " {u'answer_start': 387,\n", - " u'text': u'WatchESPN'},\n", - " {u'answer_start': 387,\n", - " u'text': u'WatchESPN'}],\n", - " u'id': u'56bec2013aeaaa14008c9372',\n", - " u'question': u'On what service could the ESPN Deportes broadcast be streamed?'},\n", - " {u'answers': [{u'answer_start': 45,\n", - " u'text': u'CBSSports.com'},\n", - " {u'answer_start': 45,\n", - " u'text': u'CBSSports.com'},\n", - " {u'answer_start': 45,\n", - " u'text': u'CBSSports'}],\n", - " u'id': u'56bec2013aeaaa14008c9373',\n", - " u'question': u'What CBS website provided a stream?'},\n", - " {u'answers': [{u'answer_start': 108,\n", - " u'text': u'Xbox One'},\n", - " {u'answer_start': 108,\n", - " u'text': u'Xbox One'},\n", - " {u'answer_start': 108,\n", - " u'text': u'Xbox One'}],\n", - " u'id': u'56bec2013aeaaa14008c9374',\n", - " u'question': u'On what game console was the CBS Sports app available?'},\n", - " {u'answers': [{u'answer_start': 104,\n", - " u'text': u'10'},\n", - " {u'answer_start': 104,\n", - " u'text': u'10'},\n", - " {u'answer_start': 104,\n", - " u'text': u'10'}],\n", - " u'id': u'56bec2013aeaaa14008c9375',\n", - " u'question': u'What version of Windows supported the CBS sports app?'},\n", - " {u'answers': [{u'answer_start': 45,\n", - " u'text': u'CBSSports.com'},\n", - " {u'answer_start': 45,\n", - " u'text': u'CBSSports.com'},\n", - " {u'answer_start': 45,\n", - " u'text': u'CBSSports.com'}],\n", - " u'id': u'56bf6e7a3aeaaa14008c961f',\n", - " u'question': u'What was the domain name of the site that streamed the Super Bowl 50 game? '},\n", - " {u'answers': [{u'answer_start': 108,\n", - " u'text': u'Xbox One'},\n", - " {u'answer_start': 108,\n", - " u'text': u'Xbox One'},\n", - " {u'answer_start': 108,\n", - " u'text': u'Xbox'}],\n", - " u'id': u'56bf6e7a3aeaaa14008c9620',\n", - " u'question': u'Which game console could viewers stream the game on? '},\n", - " {u'answers': [{u'answer_start': 269,\n", - " u'text': u'Verizon Wireless customers'},\n", - " {u'answer_start': 187,\n", - " u'text': u'Verizon'},\n", - " {u'answer_start': 187,\n", - " u'text': u'Verizon'}],\n", - " u'id': u'56bf6e7a3aeaaa14008c9621',\n", - " u'question': u'Which smartphone customers were the only people who could stream the game on their phones?'},\n", - " {u'answers': [{u'answer_start': 304,\n", - " u'text': u'NFL Mobile service'},\n", - " {u'answer_start': 64,\n", - " u'text': u'the CBS Sports apps'},\n", - " {u'answer_start': 304,\n", - " u'text': u'NFL'}],\n", - " u'id': u'56bf6e7a3aeaaa14008c9622',\n", - " u'question': u'What app did viewers use to watch the game on their smartphones? '},\n", - " {u'answers': [{u'answer_start': 187,\n", - " u'text': u'Verizon'},\n", - " {u'answer_start': 187,\n", - " u'text': u'Verizon'},\n", - " {u'answer_start': 187,\n", - " u'text': u'Verizon'}],\n", - " u'id': u'56d7205e0d65d21400198391',\n", - " u'question': u'Which wireless company had exclusive streaming rights on mobile phones?'},\n", - " {u'answers': [{u'answer_start': 304,\n", - " u'text': u'NFL Mobile service.'},\n", - " {u'answer_start': 304,\n", - " u'text': u'NFL Mobile'},\n", - " {u'answer_start': 304,\n", - " u'text': u'NFL'}],\n", - " u'id': u'56d7205e0d65d21400198392',\n", - " u'question': u'Which mobile service were Verizon customers able to watch Super Bowl 50 on their phones?'},\n", - " {u'answers': [{u'answer_start': 13,\n", - " u'text': u'digital streams of the game'},\n", - " {u'answer_start': 13,\n", - " u'text': u'digital streams'},\n", - " {u'answer_start': 13,\n", - " u'text': u'digital streams'}],\n", - " u'id': u'56d9bc13dc89441400fdb75e',\n", - " u'question': u'What did CBS provide of the Super Bowl 50 broadcast for its website, some apps and media players?'},\n", - " {u'answers': [{u'answer_start': 187,\n", - " u'text': u'Verizon'},\n", - " {u'answer_start': 187,\n", - " u'text': u'Verizon'},\n", - " {u'answer_start': 187,\n", - " u'text': u'Verizon'}],\n", - " u'id': u'56d9bc13dc89441400fdb75f',\n", - " u'question': u'What cellular carrier had the only contract for the Super Bowl to be shown on smartphones?'},\n", - " {u'answers': [{u'answer_start': 387,\n", - " u'text': u'WatchESPN.'},\n", - " {u'answer_start': 379,\n", - " u'text': u'through WatchESPN'},\n", - " {u'answer_start': 387,\n", - " u'text': u'WatchESPN'}],\n", - " u'id': u'56d9bc13dc89441400fdb760',\n", - " u'question': u'Where was the ESPN Deportes Spanish version of Super Bowl 50 available?'}]},\n", - " {u'context': u'As opposed to broadcasts of primetime series, CBS broadcast special episodes of its late night talk shows as its lead-out programs for Super Bowl 50, beginning with a special episode of The Late Show with Stephen Colbert following the game. Following a break for late local programming, CBS also aired a special episode of The Late Late Show with James Corden.',\n", - " u'qas': [{u'answers': [{u'answer_start': 186,\n", - " u'text': u'The Late Show with Stephen Colbert'},\n", - " {u'answer_start': 186,\n", - " u'text': u'The Late Show with Stephen Colbert'},\n", - " {u'answer_start': 190,\n", - " u'text': u'Late Show with Stephen Colbert'}],\n", - " u'id': u'56bec2573aeaaa14008c937b',\n", - " u'question': u'What talk show followed immediately after Super Bowl 50 on CBS?'},\n", - " {u'answers': [{u'answer_start': 323,\n", - " u'text': u'The Late Late Show with James Corden'},\n", - " {u'answer_start': 323,\n", - " u'text': u'The Late Late Show with James Corden'},\n", - " {u'answer_start': 327,\n", - " u'text': u'Late Late Show with James Corden'}],\n", - " u'id': u'56bec2573aeaaa14008c937c',\n", - " u'question': u'What show aired on CBS after late local programming?'},\n", - " {u'answers': [{u'answer_start': 186,\n", - " u'text': u'The Late Show with Stephen Colbert'},\n", - " {u'answer_start': 186,\n", - " u'text': u'The Late Show with Stephen Colbert'},\n", - " {u'answer_start': 213,\n", - " u'text': u'Colbert'}],\n", - " u'id': u'56bf6f743aeaaa14008c9631',\n", - " u'question': u'Which late night comedy host show played immediately after Super Bowl 50 ended?'},\n", - " {u'answers': [{u'answer_start': 323,\n", - " u'text': u'The Late Late Show with James Corden'},\n", - " {u'answer_start': 323,\n", - " u'text': u'The Late Late Show with James Corden'},\n", - " {u'answer_start': 327,\n", - " u'text': u'Late Late Show with James Corden'}],\n", - " u'id': u'56bf6f743aeaaa14008c9632',\n", - " u'question': u'What other CBS talk show played, after the main one that began immediately after Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 186,\n", - " u'text': u'The Late Show with Stephen Colbert'},\n", - " {u'answer_start': 186,\n", - " u'text': u'The Late Show with Stephen Colbert'},\n", - " {u'answer_start': 190,\n", - " u'text': u'Late Show with Stephen Colbert'}],\n", - " u'id': u'56d720c70d65d21400198396',\n", - " u'question': u'What CBS show followed the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 263,\n", - " u'text': u'late local programming'},\n", - " {u'answer_start': 263,\n", - " u'text': u'late local programming'},\n", - " {u'answer_start': 263,\n", - " u'text': u'late local programming'}],\n", - " u'id': u'56d720c70d65d21400198397',\n", - " u'question': u'What followed The Late Show with Stephen Colbert?'},\n", - " {u'answers': [{u'answer_start': 323,\n", - " u'text': u'The Late Late Show with James Corden.'},\n", - " {u'answer_start': 323,\n", - " u'text': u'The Late Late Show with James Corden'},\n", - " {u'answer_start': 323,\n", - " u'text': u'The Late Late Show with James Corden.'}],\n", - " u'id': u'56d720c70d65d21400198398',\n", - " u'question': u'What followed the late local programming after Super Bowl 50?'}]},\n", - " {u'context': u'CBS set the base rate for a 30-second advertisement at $5,000,000, a record high price for a Super Bowl ad. As of January 26, the advertisements had not yet sold out. CBS mandated that all advertisers purchase a package covering time on both the television and digital broadcasts of the game, meaning that for the first time, digital streams of the game would carry all national advertising in pattern with the television broadcast. This would be the final year in a multi-year contract with Anheuser-Busch InBev that allowed the beer manufacturer to air multiple advertisements during the game at a steep discount. It was also the final year that Doritos, a longtime sponsor of the game, held its \"Crash the Super Bowl\" contest that allowed viewers to create their own Doritos ads for a chance to have it aired during the game. Nintendo and The Pok\\xe9mon Company also made their Super Bowl debut, promoting the 20th anniversary of the Pok\\xe9mon video game and media franchise.',\n", - " u'qas': [{u'answers': [{u'answer_start': 55,\n", - " u'text': u'$5,000,000'},\n", - " {u'answer_start': 55,\n", - " u'text': u'$5,000,000'},\n", - " {u'answer_start': 55,\n", - " u'text': u'$5,000,000,'}],\n", - " u'id': u'56bec3153aeaaa14008c9389',\n", - " u'question': u'What was the base rate for a 30-second ad during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'},\n", - " {u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'},\n", - " {u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'}],\n", - " u'id': u'56bec3153aeaaa14008c938a',\n", - " u'question': u'What beverage company signed a contract allowing them to broadcast ads at a discount?'},\n", - " {u'answers': [{u'answer_start': 648,\n", - " u'text': u'Doritos'},\n", - " {u'answer_start': 770,\n", - " u'text': u'Doritos'},\n", - " {u'answer_start': 648,\n", - " u'text': u'Doritos'}],\n", - " u'id': u'56bec3153aeaaa14008c938b',\n", - " u'question': u'What brand sponsored the \"Crash the Super Bowl\" contest?'},\n", - " {u'answers': [{u'answer_start': 910,\n", - " u'text': u'20th'},\n", - " {u'answer_start': 906,\n", - " u'text': u'the 20th'},\n", - " {u'answer_start': 910,\n", - " u'text': u'20th'}],\n", - " u'id': u'56bec3153aeaaa14008c938c',\n", - " u'question': u'What anniversary of the Pok\\xe9mon franchise was celebrated during the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 55,\n", - " u'text': u'$5,000,000'},\n", - " {u'answer_start': 55,\n", - " u'text': u'$5,000,000'},\n", - " {u'answer_start': 55,\n", - " u'text': u'$5,000,000'}],\n", - " u'id': u'56bf717d3aeaaa14008c9635',\n", - " u'question': u'What was the starting cost for a TV commercial lasting 30 seconds and airing during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'},\n", - " {u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'},\n", - " {u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'}],\n", - " u'id': u'56bf717d3aeaaa14008c9637',\n", - " u'question': u'Which company was given permission to air TV commercials during Super Bowl 50 at a discounted price?'},\n", - " {u'answers': [{u'answer_start': 648,\n", - " u'text': u'Doritos'},\n", - " {u'answer_start': 770,\n", - " u'text': u'Doritos'},\n", - " {u'answer_start': 648,\n", - " u'text': u'Doritos'}],\n", - " u'id': u'56bf717d3aeaaa14008c9638',\n", - " u'question': u'Which company held a contest whereupon contestants could enter for a chance to have their own commercial shown during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 829,\n", - " u'text': u'Nintendo'},\n", - " {u'answer_start': 829,\n", - " u'text': u'Nintendo'},\n", - " {u'answer_start': 829,\n", - " u'text': u'Nintendo'}],\n", - " u'id': u'56bf717d3aeaaa14008c9639',\n", - " u'question': u'Which video gaming company debuted their ad for the first time during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 842,\n", - " u'text': u'The Pok\\xe9mon Company'},\n", - " {u'answer_start': 842,\n", - " u'text': u'The Pok\\xe9mon Company'},\n", - " {u'answer_start': 842,\n", - " u'text': u'The Pok\\xe9mon Company'}],\n", - " u'id': u'56d721af0d65d2140019839e',\n", - " u'question': u'Who made their Super Bowl commercial debut with Nintendo?'},\n", - " {u'answers': [{u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'},\n", - " {u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'},\n", - " {u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'}],\n", - " u'id': u'56d721af0d65d2140019839f',\n", - " u'question': u'Which company has been able to air multiple ads at a steep discount?'},\n", - " {u'answers': [{u'answer_start': 648,\n", - " u'text': u'Doritos'},\n", - " {u'answer_start': 648,\n", - " u'text': u'Doritos'},\n", - " {u'answer_start': 648,\n", - " u'text': u'Doritos'}],\n", - " u'id': u'56d721af0d65d214001983a0',\n", - " u'question': u'Which company has held contests for fans to create their own ad for the company?'},\n", - " {u'answers': [{u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'},\n", - " {u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'},\n", - " {u'answer_start': 492,\n", - " u'text': u'Anheuser-Busch InBev'}],\n", - " u'id': u'56d9bdc1dc89441400fdb768',\n", - " u'question': u'What company was in its last year for getting Super Bowl commercials at a big discount?'},\n", - " {u'answers': [{u'answer_start': 648,\n", - " u'text': u'Doritos'},\n", - " {u'answer_start': 648,\n", - " u'text': u'Doritos'},\n", - " {u'answer_start': 648,\n", - " u'text': u'Doritos'}],\n", - " u'id': u'56d9bdc1dc89441400fdb769',\n", - " u'question': u'What company was this the last Super Bowl they would let customers have an opportunity to get their own commercial ideas aired?'},\n", - " {u'answers': [{u'answer_start': 699,\n", - " u'text': u'Crash the Super Bowl'},\n", - " {u'answer_start': 699,\n", - " u'text': u'Crash the Super Bowl'},\n", - " {u'answer_start': 699,\n", - " u'text': u'Crash the Super Bowl'}],\n", - " u'id': u'56d9bdc1dc89441400fdb76a',\n", - " u'question': u'What was the Doritos customer Super Bowl ad campaign called?'}]},\n", - " {u'context': u'QuickBooks sponsored a \"Small Business Big Game\" contest, in which Death Wish Coffee had a 30-second commercial aired free of charge courtesy of QuickBooks. Death Wish Coffee beat out nine other contenders from across the United States for the free advertisement.',\n", - " u'qas': [{u'answers': [{u'answer_start': 23,\n", - " u'text': u'\"Small Business Big Game\"'},\n", - " {u'answer_start': 24,\n", - " u'text': u'Small Business Big Game'},\n", - " {u'answer_start': 24,\n", - " u'text': u'Small Business Big Game'}],\n", - " u'id': u'56bec38b3aeaaa14008c9397',\n", - " u'question': u'What was the name of the contest sponsored by QuickBooks?'},\n", - " {u'answers': [{u'answer_start': 67,\n", - " u'text': u'Death Wish Coffee'},\n", - " {u'answer_start': 67,\n", - " u'text': u'Death Wish Coffee'},\n", - " {u'answer_start': 157,\n", - " u'text': u'Death Wish Coffee'}],\n", - " u'id': u'56bec38b3aeaaa14008c9398',\n", - " u'question': u'What company won a free advertisement due to the QuickBooks contest?'},\n", - " {u'answers': [{u'answer_start': 91,\n", - " u'text': u'30-second'},\n", - " {u'answer_start': 91,\n", - " u'text': u'30-second'},\n", - " {u'answer_start': 91,\n", - " u'text': u'30-second'}],\n", - " u'id': u'56bec38b3aeaaa14008c9399',\n", - " u'question': u'How long was the Death Wish Coffee commercial?'},\n", - " {u'answers': [{u'answer_start': 184,\n", - " u'text': u'nine'},\n", - " {u'answer_start': 184,\n", - " u'text': u'nine'},\n", - " {u'answer_start': 184,\n", - " u'text': u'nine'}],\n", - " u'id': u'56bec38b3aeaaa14008c939a',\n", - " u'question': u'Besides Death Wish Coffee, how many other competitors participated in the contest?'},\n", - " {u'answers': [{u'answer_start': 67,\n", - " u'text': u'Death Wish Coffee'},\n", - " {u'answer_start': 67,\n", - " u'text': u'Death Wish Coffee'},\n", - " {u'answer_start': 157,\n", - " u'text': u'Death Wish Coffee'}],\n", - " u'id': u'56bf71e73aeaaa14008c963f',\n", - " u'question': u'Which company won a contest to have their ad shown for free during Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 184,\n", - " u'text': u'nine'},\n", - " {u'answer_start': 184,\n", - " u'text': u'nine'},\n", - " {u'answer_start': 184,\n", - " u'text': u'nine'}],\n", - " u'id': u'56bf71e73aeaaa14008c9640',\n", - " u'question': u'How many other contestants did the company, that had their ad shown for free, beat out?'},\n", - " {u'answers': [{u'answer_start': 145,\n", - " u'text': u'QuickBooks.'},\n", - " {u'answer_start': 0,\n", - " u'text': u'QuickBooks'},\n", - " {u'answer_start': 0,\n", - " u'text': u'QuickBooks'}],\n", - " u'id': u'56d722330d65d214001983a6',\n", - " u'question': u'Which company sponsored a contest called \"Small Business Big Game\"?'},\n", - " {u'answers': [{u'answer_start': 67,\n", - " u'text': u'Death Wish Coffee'},\n", - " {u'answer_start': 67,\n", - " u'text': u'Death Wish Coffee'},\n", - " {u'answer_start': 157,\n", - " u'text': u'Death Wish Coffee'}],\n", - " u'id': u'56d722330d65d214001983a7',\n", - " u'question': u'Which company had a commercial aired for free as a result of the Quickbooks contest?'},\n", - " {u'answers': [{u'answer_start': 198,\n", - " u'text': u'ten'},\n", - " {u'answer_start': 184,\n", - " u'text': u'nine'},\n", - " {u'answer_start': 184,\n", - " u'text': u'nine'}],\n", - " u'id': u'56d722330d65d214001983a8',\n", - " u'question': u'How many companies were part of the Quickbooks contest?'},\n", - " {u'answers': [{u'answer_start': 145,\n", - " u'text': u'QuickBooks.'},\n", - " {u'answer_start': 0,\n", - " u'text': u'QuickBooks'},\n", - " {u'answer_start': 0,\n", - " u'text': u'QuickBooks'}],\n", - " u'id': u'56d9be16dc89441400fdb770',\n", - " u'question': u'What company had a contest to win a free Super Bowl commercial?'},\n", - " {u'answers': [{u'answer_start': 67,\n", - " u'text': u'Death Wish Coffee'},\n", - " {u'answer_start': 67,\n", - " u'text': u'Death Wish Coffee'},\n", - " {u'answer_start': 157,\n", - " u'text': u'Death Wish Coffee'}],\n", - " u'id': u'56d9be16dc89441400fdb771',\n", - " u'question': u'Who won the competition to get a free Super Bowl commercial aired?'}]},\n", - " {u'context': u'20th Century Fox, Lionsgate, Paramount Pictures, Universal Studios and Walt Disney Studios paid for movie trailers to be aired during the Super Bowl. Fox paid for Deadpool, X-Men: Apocalypse, Independence Day: Resurgence and Eddie the Eagle, Lionsgate paid for Gods of Egypt, Paramount paid for Teenage Mutant Ninja Turtles: Out of the Shadows and 10 Cloverfield Lane, Universal paid for The Secret Life of Pets and the debut trailer for Jason Bourne and Disney paid for Captain America: Civil War, The Jungle Book and Alice Through the Looking Glass.[citation needed]',\n", - " u'qas': [{u'answers': [{u'answer_start': 438,\n", - " u'text': u'Jason Bourne'},\n", - " {u'answer_start': 438,\n", - " u'text': u'Jason Bourne'},\n", - " {u'answer_start': 438,\n", - " u'text': u'Jason Bourne'}],\n", - " u'id': u'56bec4343aeaaa14008c93a9',\n", - " u'question': u'What Universal trailer debuted during the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 261,\n", - " u'text': u'Gods of Egypt'},\n", - " {u'answer_start': 261,\n", - " u'text': u'Gods of Egypt'},\n", - " {u'answer_start': 261,\n", - " u'text': u'Gods of Egypt,'}],\n", - " u'id': u'56bec4343aeaaa14008c93aa',\n", - " u'question': u'What Lionsgate trailer was shown during the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 295,\n", - " u'text': u'Teenage Mutant Ninja Turtles: Out of the Shadows'},\n", - " {u'answer_start': 295,\n", - " u'text': u'Teenage Mutant Ninja Turtles: Out of the Shadows'},\n", - " {u'answer_start': 295,\n", - " u'text': u'Teenage Mutant Ninja Turtles: Out of the Shadows'}],\n", - " u'id': u'56bec4343aeaaa14008c93ab',\n", - " u'question': u'Along with 10 Cloverfield Lane, what Paramount trailer appeared during the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 210,\n", - " u'text': u'Resurgence'},\n", - " {u'answer_start': 210,\n", - " u'text': u'Resurgence'},\n", - " {u'answer_start': 210,\n", - " u'text': u'Resurgence'}],\n", - " u'id': u'56bec4343aeaaa14008c93ac',\n", - " u'question': u'What was the subtitle of the Independence Day movie that had a trailer during the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 261,\n", - " u'text': u'Gods of Egypt'},\n", - " {u'answer_start': 261,\n", - " u'text': u'Gods of Egypt'},\n", - " {u'answer_start': 261,\n", - " u'text': u'Gods of Egypt'}],\n", - " u'id': u'56d723560d65d214001983ac',\n", - " u'question': u'What film did Lionsgate pay to have the trailer aired during the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 295,\n", - " u'text': u'Teenage Mutant Ninja Turtles: Out of the Shadows'},\n", - " {u'answer_start': 295,\n", - " u'text': u'Teenage Mutant Ninja Turtles: Out of the Shadows'},\n", - " {u'answer_start': 295,\n", - " u'text': u'Teenage Mutant Ninja Turtles'}],\n", - " u'id': u'56d723560d65d214001983ad',\n", - " u'question': u'Paramount paid fo, 10 Cloverfield Lane and which other film trailer to be aired during the game?'},\n", - " {u'answers': [{u'answer_start': 438,\n", - " u'text': u'Jason Bourne'},\n", - " {u'answer_start': 438,\n", - " u'text': u'Jason Bourne'},\n", - " {u'answer_start': 438,\n", - " u'text': u'Jason Bourne'}],\n", - " u'id': u'56d723560d65d214001983ae',\n", - " u'question': u'Universal Paid for The Secret Life of Pets and which other film trailer?'},\n", - " {u'answers': [{u'answer_start': 471,\n", - " u'text': u'Captain America: Civil War'},\n", - " {u'answer_start': 471,\n", - " u'text': u'Captain America: Civil War'},\n", - " {u'answer_start': 471,\n", - " u'text': u'Captain America: Civil War'}],\n", - " u'id': u'56d723560d65d214001983af',\n", - " u'question': u'Disney paid for The Jungle Book, Alice Through the Looking Glass, and which other film trailer to be aired during the game?'},\n", - " {u'answers': [{u'answer_start': 192,\n", - " u'text': u'Independence Day'},\n", - " {u'answer_start': 192,\n", - " u'text': u'Independence Day'},\n", - " {u'answer_start': 192,\n", - " u'text': u'Independence Day: Resurgence'}],\n", - " u'id': u'56d9bf70dc89441400fdb77b',\n", - " u'question': u'What famous July Fourth holiday movie did Fox pay to advertise a sequel of during the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 49,\n", - " u'text': u'Universal'},\n", - " {u'answer_start': 369,\n", - " u'text': u'Universal'},\n", - " {u'answer_start': 369,\n", - " u'text': u'Universal'}],\n", - " u'id': u'56d9bf70dc89441400fdb77c',\n", - " u'question': u'What movie company paid to have the next Jason Bourne movie ad shown during the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 13,\n", - " u'text': u'Fox'},\n", - " {u'answer_start': 150,\n", - " u'text': u'Fox'},\n", - " {u'answer_start': 455,\n", - " u'text': u'Disney'}],\n", - " u'id': u'56d9bf70dc89441400fdb77d',\n", - " u'question': u'What company paid for a Super Bowl 50 ad to show a trailer of X-Men: Apocalypse?'}]},\n", - " {u'context': u'Westwood One will carry the game throughout North America, with Kevin Harlan as play-by-play announcer, Boomer Esiason and Dan Fouts as color analysts, and James Lofton and Mark Malone as sideline reporters. Jim Gray will anchor the pre-game and halftime coverage.',\n", - " u'qas': [{u'answers': [{u'answer_start': 0,\n", - " u'text': u'Westwood One'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Westwood One'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Westwood One'}],\n", - " u'id': u'56bec4a33aeaaa14008c93b1',\n", - " u'question': u'What radio network carried the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'},\n", - " {u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'},\n", - " {u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'}],\n", - " u'id': u'56bec4a33aeaaa14008c93b2',\n", - " u'question': u'Who did play-by-play on Westwood One?'},\n", - " {u'answers': [{u'answer_start': 208,\n", - " u'text': u'Jim Gray'},\n", - " {u'answer_start': 208,\n", - " u'text': u'Jim Gray'},\n", - " {u'answer_start': 206,\n", - " u'text': u'. Jim Gray'}],\n", - " u'id': u'56bec4a33aeaaa14008c93b3',\n", - " u'question': u'Who handled pre-game coverage on Westwood One?'},\n", - " {u'answers': [{u'answer_start': 104,\n", - " u'text': u'Boomer Esiason'},\n", - " {u'answer_start': 104,\n", - " u'text': u'Boomer Esiason'},\n", - " {u'answer_start': 104,\n", - " u'text': u'Boomer Esiason'}],\n", - " u'id': u'56bec4a33aeaaa14008c93b4',\n", - " u'question': u'Along with Dan Fouts, who served as a color analyst for the radio broadcast?'},\n", - " {u'answers': [{u'answer_start': 156,\n", - " u'text': u'James Lofton'},\n", - " {u'answer_start': 156,\n", - " u'text': u'James Lofton'},\n", - " {u'answer_start': 156,\n", - " u'text': u'James Lofton and Mark Malone'}],\n", - " u'id': u'56bec4a33aeaaa14008c93b5',\n", - " u'question': u'Who shared sideline reporting duties with Mark Malone?'},\n", - " {u'answers': [{u'answer_start': 3,\n", - " u'text': u'two'},\n", - " {u'answer_start': 104,\n", - " u'text': u'Boomer Esiason and Dan Fouts'}],\n", - " u'id': u'56bf738b3aeaaa14008c9655',\n", - " u'question': u'How many color analysts were involved with Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'},\n", - " {u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'},\n", - " {u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'}],\n", - " u'id': u'56bf738b3aeaaa14008c9656',\n", - " u'question': u'Who announced the game play-by-play for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'Westwood One'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Westwood One'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Westwood One'}],\n", - " u'id': u'56d723ad0d65d214001983b4',\n", - " u'question': u'Who will carry the game throughout all of North America?'},\n", - " {u'answers': [{u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'},\n", - " {u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'},\n", - " {u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'}],\n", - " u'id': u'56d723ad0d65d214001983b5',\n", - " u'question': u'Who is the play-by-play announcer for the game?'},\n", - " {u'answers': [{u'answer_start': 208,\n", - " u'text': u'Jim Gray'},\n", - " {u'answer_start': 208,\n", - " u'text': u'Jim Gray'},\n", - " {u'answer_start': 208,\n", - " u'text': u'Jim Gray'}],\n", - " u'id': u'56d723ad0d65d214001983b8',\n", - " u'question': u'Who covered the halftime coverage?'},\n", - " {u'answers': [{u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'},\n", - " {u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'},\n", - " {u'answer_start': 64,\n", - " u'text': u'Kevin Harlan'}],\n", - " u'id': u'56d9c049dc89441400fdb78e',\n", - " u'question': u\"Who was the announcer for Westwood One's Super Bowl 50 coverage?\"},\n", - " {u'answers': [{u'answer_start': 104,\n", - " u'text': u'Boomer Esiason and Dan Fouts'},\n", - " {u'answer_start': 104,\n", - " u'text': u'Boomer Esiason and Dan Fouts'},\n", - " {u'answer_start': 104,\n", - " u'text': u'Boomer Esiason and Dan Fouts'}],\n", - " u'id': u'56d9c049dc89441400fdb78f',\n", - " u'question': u'Who were the Westwood one color analysts?'},\n", - " {u'answers': [{u'answer_start': 156,\n", - " u'text': u'James Lofton and Mark Malone'},\n", - " {u'answer_start': 156,\n", - " u'text': u'James Lofton and Mark Malone'},\n", - " {u'answer_start': 156,\n", - " u'text': u'James Lofton and Mark Malone'}],\n", - " u'id': u'56d9c049dc89441400fdb790',\n", - " u'question': u'Who were the Westwood One sideline announcers?'},\n", - " {u'answers': [{u'answer_start': 233,\n", - " u'text': u'pre-game and halftime coverage.'},\n", - " {u'answer_start': 246,\n", - " u'text': u'halftime'},\n", - " {u'answer_start': 222,\n", - " u'text': u'anchor the pre-game and halftime coverage'}],\n", - " u'id': u'56d9c049dc89441400fdb791',\n", - " u'question': u'What did Jim Gray cover?'},\n", - " {u'answers': [{u'answer_start': 44,\n", - " u'text': u'North America'},\n", - " {u'answer_start': 44,\n", - " u'text': u'North America'},\n", - " {u'answer_start': 44,\n", - " u'text': u'North America'}],\n", - " u'id': u'56d9c049dc89441400fdb792',\n", - " u'question': u'What area did the Westwood One broadcast cover?'}]},\n", - " {u'context': u'The flagship stations of each station in the markets of each team will carry their local play-by-play calls. In Denver, KOA (850 AM) and KRFX (103.5 FM) will carry the game, with Dave Logan on play-by-play and Ed McCaffrey on color commentary. In North Carolina, WBT (1110 AM) will carry the game, with Mick Mixon on play-by-play and Eugene Robinson and Jim Szoke on color commentary. WBT will also simulcast the game on its sister station WBT-FM (99.3 FM), which is based in Chester, South Carolina. As KOA and WBT are both clear-channel stations, the local broadcasts will be audible over much of the western United States after sunset (for Denver) and the eastern United States throughout the game (for Carolina). In accordance with contractual rules, the rest of the stations in the Broncos and Panthers radio networks will either carry the Westwood One feed or not carry the game at all.',\n", - " u'qas': [{u'answers': [{u'answer_start': 137,\n", - " u'text': u'KRFX'},\n", - " {u'answer_start': 137,\n", - " u'text': u'KRFX'},\n", - " {u'answer_start': 137,\n", - " u'text': u'KRFX'}],\n", - " u'id': u'56bec5233aeaaa14008c93cf',\n", - " u'question': u'What radio station is located at 103.5FM on the dial in Denver?'},\n", - " {u'answers': [{u'answer_start': 179,\n", - " u'text': u'Dave Logan'},\n", - " {u'answer_start': 120,\n", - " u'text': u'KOA (850 AM) and KRFX (103.5 FM)'},\n", - " {u'answer_start': 179,\n", - " u'text': u'Dave Logan'}],\n", - " u'id': u'56bec5233aeaaa14008c93d0',\n", - " u'question': u'Who handled play-by-play for the Denver radio market?'},\n", - " {u'answers': [{u'answer_start': 268,\n", - " u'text': u'1110 AM'},\n", - " {u'answer_start': 268,\n", - " u'text': u'1110 AM'},\n", - " {u'answer_start': 267,\n", - " u'text': u'(1110 AM)'}],\n", - " u'id': u'56bec5233aeaaa14008c93d1',\n", - " u'question': u'What is the frequency of the radio station WBT in North Carolina?'},\n", - " {u'answers': [{u'answer_start': 476,\n", - " u'text': u'Chester, South Carolina'},\n", - " {u'answer_start': 476,\n", - " u'text': u'Chester'},\n", - " {u'answer_start': 476,\n", - " u'text': u'Chester, South Carolina'}],\n", - " u'id': u'56bec5233aeaaa14008c93d2',\n", - " u'question': u'In what city is WBT-FM located?'},\n", - " {u'answers': [{u'answer_start': 303,\n", - " u'text': u'Mick Mixon'},\n", - " {u'answer_start': 303,\n", - " u'text': u'Mick Mixon'},\n", - " {u'answer_start': 303,\n", - " u'text': u'Mick Mixon'}],\n", - " u'id': u'56bec5233aeaaa14008c93d3',\n", - " u'question': u'Who did play-by-play for the WBT broadcast?'},\n", - " {u'answers': [{u'answer_start': 179,\n", - " u'text': u'Dave Logan'},\n", - " {u'answer_start': 179,\n", - " u'text': u'Dave Logan'},\n", - " {u'answer_start': 179,\n", - " u'text': u'Dave Logan'}],\n", - " u'id': u'56d724620d65d214001983be',\n", - " u'question': u'Who handled the play-by-play for the Denver radio stations?'},\n", - " {u'answers': [{u'answer_start': 210,\n", - " u'text': u'Ed McCaffrey'},\n", - " {u'answer_start': 210,\n", - " u'text': u'Ed McCaffrey'},\n", - " {u'answer_start': 210,\n", - " u'text': u'Ed McCaffrey'}],\n", - " u'id': u'56d724620d65d214001983bf',\n", - " u'question': u\"Who handled the color commentary for Denver's radio stations?\"},\n", - " {u'answers': [{u'answer_start': 263,\n", - " u'text': u'WBT'},\n", - " {u'answer_start': 263,\n", - " u'text': u'WBT (1110 AM)'},\n", - " {u'answer_start': 263,\n", - " u'text': u'WBT (1110 AM)'}],\n", - " u'id': u'56d724620d65d214001983c0',\n", - " u'question': u'Which station covered the game for North Carolina?'},\n", - " {u'answers': [{u'answer_start': 303,\n", - " u'text': u'Mick Mixon'},\n", - " {u'answer_start': 303,\n", - " u'text': u'Mick Mixon'},\n", - " {u'answer_start': 303,\n", - " u'text': u'Mick Mixon'}],\n", - " u'id': u'56d724620d65d214001983c1',\n", - " u'question': u'Who handled the play-by-play for WBT?'},\n", - " {u'answers': [{u'answer_start': 120,\n", - " u'text': u'KOA (850 AM) and KRFX (103.5 FM)'},\n", - " {u'answer_start': 120,\n", - " u'text': u'KOA (850 AM) and KRFX (103.5 FM)'},\n", - " {u'answer_start': 120,\n", - " u'text': u'KOA (850 AM) and KRFX (103.5 FM)'}],\n", - " u'id': u'56d9c203dc89441400fdb7a0',\n", - " u'question': u'What two radio stations in Denver carried Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 440,\n", - " u'text': u'WBT-FM (99.3 FM)'},\n", - " {u'answer_start': 263,\n", - " u'text': u'WBT (1110 AM)'},\n", - " {u'answer_start': 440,\n", - " u'text': u'WBT-FM'}],\n", - " u'id': u'56d9c203dc89441400fdb7a3',\n", - " u'question': u'What radio station in South Carolina carried the Super Bowl 50 game?'}]},\n", - " {u'context': u'In the United Kingdom, BBC Radio 5 Live and 5 Live Sports Extra will carry the contest. The BBC will carry its own British English broadcast, with Greg Brady, Darren Fletcher and Rocky Boiman on commentary.',\n", - " u'qas': [{u'answers': [{u'answer_start': 23,\n", - " u'text': u'BBC Radio 5'},\n", - " {u'answer_start': 27,\n", - " u'text': u'Radio 5 Live'},\n", - " {u'answer_start': 23,\n", - " u'text': u'BBC Radio 5 Live'}],\n", - " u'id': u'56bec5ff3aeaaa14008c93e3',\n", - " u'question': u'What BBC radio station will carry the game in the United Kingdom?'},\n", - " {u'answers': [{u'answer_start': 44,\n", - " u'text': u'5 Live Sports Extra'},\n", - " {u'answer_start': 44,\n", - " u'text': u'5 Live Sports Extra'},\n", - " {u'answer_start': 44,\n", - " u'text': u'5 Live Sports Extra'}],\n", - " u'id': u'56bec5ff3aeaaa14008c93e4',\n", - " u'question': u'Aside from BBC Radio 5, what radio station will broadcast the game?'},\n", - " {u'answers': [{u'answer_start': 159,\n", - " u'text': u'Darren Fletcher'},\n", - " {u'answer_start': 159,\n", - " u'text': u'Darren Fletcher'},\n", - " {u'answer_start': 159,\n", - " u'text': u'Darren Fletcher'}],\n", - " u'id': u'56bec5ff3aeaaa14008c93e5',\n", - " u'question': u'Who makes up the BBC commentary team with Greg Brady and Rocky Boiman?'},\n", - " {u'answers': [{u'answer_start': 23,\n", - " u'text': u'BBC'},\n", - " {u'answer_start': 23,\n", - " u'text': u'BBC'},\n", - " {u'answer_start': 92,\n", - " u'text': u'BBC'}],\n", - " u'id': u'56d724ea0d65d214001983c8',\n", - " u'question': u'What British TV network will carry the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 147,\n", - " u'text': u'Greg Brady'},\n", - " {u'answer_start': 147,\n", - " u'text': u'Greg Brady'},\n", - " {u'answer_start': 147,\n", - " u'text': u'Greg Brady'}],\n", - " u'id': u'56d724ea0d65d214001983c9',\n", - " u'question': u'British commentators include, Darren Fletcher, Rocky Boiman and who else?'}]},\n", - " {u'context': u'In honor of the 50th Super Bowl, the pregame ceremony featured the on-field introduction of 39 of the 43 previous Super Bowl Most Valuable Players. Bart Starr (MVP of Super Bowls I and II) and Chuck Howley (MVP of Super Bowl V) appeared via video, while Peyton Manning (MVP of Super Bowl XLI and current Broncos quarterback) was shown in the locker room preparing for the game. No plans were announced regarding the recognition of Harvey Martin, co-MVP of Super Bowl XII, who died in 2001.',\n", - " u'qas': [{u'answers': [{u'answer_start': 148,\n", - " u'text': u'Bart Starr'},\n", - " {u'answer_start': 148,\n", - " u'text': u'Bart Starr'},\n", - " {u'answer_start': 153,\n", - " u'text': u'Starr'}],\n", - " u'id': u'56bec6573aeaaa14008c93e9',\n", - " u'question': u'Who was the Most Valuable Player of Super Bowl II?'},\n", - " {u'answers': [{u'answer_start': 193,\n", - " u'text': u'Chuck Howley'},\n", - " {u'answer_start': 193,\n", - " u'text': u'Chuck Howley'},\n", - " {u'answer_start': 199,\n", - " u'text': u'Howley'}],\n", - " u'id': u'56bec6573aeaaa14008c93ea',\n", - " u'question': u'What player won the MVP in Super Bowl V?'},\n", - " {u'answers': [{u'answer_start': 254,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 254,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 261,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56bec6573aeaaa14008c93eb',\n", - " u'question': u'Who was the Most Valuable Player of Super Bowl XLI?'},\n", - " {u'answers': [{u'answer_start': 484,\n", - " u'text': u'2001'},\n", - " {u'answer_start': 484,\n", - " u'text': u'2001'},\n", - " {u'answer_start': 484,\n", - " u'text': u'2001'}],\n", - " u'id': u'56bec6573aeaaa14008c93ec',\n", - " u'question': u'In what year did Harvey Martin die?'},\n", - " {u'answers': [{u'answer_start': 254,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 254,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 261,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56bec6573aeaaa14008c93ed',\n", - " u'question': u'Who is the present quarterback of the Broncos?'},\n", - " {u'answers': [{u'answer_start': 92,\n", - " u'text': u'39'},\n", - " {u'answer_start': 92,\n", - " u'text': u'39'},\n", - " {u'answer_start': 92,\n", - " u'text': u'39'}],\n", - " u'id': u'56d7251d0d65d214001983cc',\n", - " u'question': u'How many of the prior Super Bowl MVPs appeared together at the pregame show?'},\n", - " {u'answers': [{u'answer_start': 254,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 254,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 261,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56d7251d0d65d214001983cd',\n", - " u'question': u'What former MVP was shown in the locker room?'},\n", - " {u'answers': [{u'answer_start': 431,\n", - " u'text': u'Harvey Martin'},\n", - " {u'answer_start': 431,\n", - " u'text': u'Harvey Martin'},\n", - " {u'answer_start': 438,\n", - " u'text': u'Martin'}],\n", - " u'id': u'56d7251d0d65d214001983cf',\n", - " u'question': u'Which MVP died in 2001?'},\n", - " {u'answers': [{u'answer_start': 102,\n", - " u'text': u'43'},\n", - " {u'answer_start': 102,\n", - " u'text': u'43'},\n", - " {u'answer_start': 102,\n", - " u'text': u'43'}],\n", - " u'id': u'56d9c3a6dc89441400fdb7b4',\n", - " u'question': u'How many players have been awarded the Most Valuable Player distinction for the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 92,\n", - " u'text': u'39'},\n", - " {u'answer_start': 92,\n", - " u'text': u'39'},\n", - " {u'answer_start': 92,\n", - " u'text': u'39'}],\n", - " u'id': u'56d9c3a6dc89441400fdb7b5',\n", - " u'question': u'How many former MVP honorees were present for a pregame ceremony?'},\n", - " {u'answers': [{u'answer_start': 148,\n", - " u'text': u'Bart Starr'},\n", - " {u'answer_start': 148,\n", - " u'text': u'Bart Starr'},\n", - " {u'answer_start': 153,\n", - " u'text': u'Starr'}],\n", - " u'id': u'56d9c3a6dc89441400fdb7b6',\n", - " u'question': u'Who was the MVP of Super Bowl I and II?'},\n", - " {u'answers': [{u'answer_start': 254,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 254,\n", - " u'text': u'Peyton Manning'},\n", - " {u'answer_start': 261,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56d9c3a6dc89441400fdb7b7',\n", - " u'question': u'Who was the MVP of Super Bowl XLI?'},\n", - " {u'answers': [{u'answer_start': 431,\n", - " u'text': u'Harvey Martin'},\n", - " {u'answer_start': 431,\n", - " u'text': u'Harvey Martin'},\n", - " {u'answer_start': 431,\n", - " u'text': u'Harvey Martin'}],\n", - " u'id': u'56d9c3a6dc89441400fdb7b8',\n", - " u'question': u'Which MVP honoree died in 2001?'}]},\n", - " {u'context': u'Six-time Grammy winner and Academy Award nominee Lady Gaga performed the national anthem, while Academy Award winner Marlee Matlin provided American Sign Language (ASL) translation.',\n", - " u'qas': [{u'answers': [{u'answer_start': 0,\n", - " u'text': u'Six'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Six'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Six'}],\n", - " u'id': u'56bec6ac3aeaaa14008c93fd',\n", - " u'question': u'How many Grammys has Lady Gaga won?'},\n", - " {u'answers': [{u'answer_start': 69,\n", - " u'text': u'the national anthem'},\n", - " {u'answer_start': 69,\n", - " u'text': u'the national anthem'},\n", - " {u'answer_start': 73,\n", - " u'text': u'national anthem'}],\n", - " u'id': u'56bec6ac3aeaaa14008c93fe',\n", - " u'question': u'What did Lady Gaga sing?'},\n", - " {u'answers': [{u'answer_start': 27,\n", - " u'text': u'Academy Award'},\n", - " {u'answer_start': 96,\n", - " u'text': u'Academy Award'},\n", - " {u'answer_start': 96,\n", - " u'text': u'Academy Award'}],\n", - " u'id': u'56bec6ac3aeaaa14008c93ff',\n", - " u'question': u'What award has Marlee Matlin won?'},\n", - " {u'answers': [{u'answer_start': 69,\n", - " u'text': u'the national anthem'},\n", - " {u'answer_start': 69,\n", - " u'text': u'the national anthem'},\n", - " {u'answer_start': 73,\n", - " u'text': u'national anthem'}],\n", - " u'id': u'56bec6ac3aeaaa14008c9400',\n", - " u'question': u'What did Marlee Matlin translate?'},\n", - " {u'answers': [{u'answer_start': 140,\n", - " u'text': u'American Sign Language'},\n", - " {u'answer_start': 140,\n", - " u'text': u'American Sign Language'},\n", - " {u'answer_start': 140,\n", - " u'text': u'American Sign Language'}],\n", - " u'id': u'56bec6ac3aeaaa14008c9401',\n", - " u'question': u'Into what language did Marlee Matlin translate the national anthem?'},\n", - " {u'answers': [{u'answer_start': 49,\n", - " u'text': u'Lady Gaga'},\n", - " {u'answer_start': 49,\n", - " u'text': u'Lady Gaga'},\n", - " {u'answer_start': 49,\n", - " u'text': u'Lady Gaga'}],\n", - " u'id': u'56d20650e7d4791d00902614',\n", - " u'question': u'Who sang the national anthem?'},\n", - " {u'answers': [{u'answer_start': 117,\n", - " u'text': u'Marlee Matlin'},\n", - " {u'answer_start': 117,\n", - " u'text': u'Marlee Matlin'},\n", - " {u'answer_start': 117,\n", - " u'text': u'Marlee Matlin'}],\n", - " u'id': u'56d20650e7d4791d00902615',\n", - " u'question': u'What actress did the ASL translation for the game?'},\n", - " {u'answers': [{u'answer_start': 49,\n", - " u'text': u'Lady Gaga'},\n", - " {u'answer_start': 49,\n", - " u'text': u'Lady Gaga'},\n", - " {u'answer_start': 49,\n", - " u'text': u'Lady Gaga'}],\n", - " u'id': u'56d7253b0d65d214001983d4',\n", - " u'question': u'Who did the National Anthem at Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 117,\n", - " u'text': u'Marlee Matlin'},\n", - " {u'answer_start': 117,\n", - " u'text': u'Marlee Matlin'},\n", - " {u'answer_start': 117,\n", - " u'text': u'Marlee Matlin'}],\n", - " u'id': u'56d7253b0d65d214001983d5',\n", - " u'question': u'What actor did sign language for the National Anthem at Superbowl 50?'},\n", - " {u'answers': [{u'answer_start': 49,\n", - " u'text': u'Lady Gaga'},\n", - " {u'answer_start': 49,\n", - " u'text': u'Lady Gaga'},\n", - " {u'answer_start': 49,\n", - " u'text': u'Lady Gaga'}],\n", - " u'id': u'56d9c455dc89441400fdb7c4',\n", - " u'question': u'Who did the Super Bowl 50 National Anthem?'},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'Six'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Six'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Six'}],\n", - " u'id': u'56d9c455dc89441400fdb7c5',\n", - " u'question': u\"How many Grammy Award's does the text say Lady Gaga has won?\"},\n", - " {u'answers': [{u'answer_start': 117,\n", - " u'text': u'Marlee Matlin'},\n", - " {u'answer_start': 117,\n", - " u'text': u'Marlee Matlin'},\n", - " {u'answer_start': 117,\n", - " u'text': u'Marlee Matlin'}],\n", - " u'id': u'56d9c455dc89441400fdb7c6',\n", - " u'question': u'Who did the sign language of the National Anthem at Super Bowl 50?'}]},\n", - " {u'context': u'In late November 2015, reports surfaced stating that \"multiple acts\" would perform during the halftime show. On December 3, the league confirmed that the show would be headlined by the British rock group Coldplay. On January 7, 2016, Pepsi confirmed to the Associated Press that Beyonc\\xe9, who headlined the Super Bowl XLVII halftime show and collaborated with Coldplay on the single \"Hymn for the Weekend\", would be making an appearance. Bruno Mars, who headlined the Super Bowl XLVIII halftime show, and Mark Ronson also performed.',\n", - " u'qas': [{u'answers': [{u'answer_start': 112,\n", - " u'text': u'December 3'},\n", - " {u'answer_start': 112,\n", - " u'text': u'December 3'},\n", - " {u'answer_start': 112,\n", - " u'text': u'December 3'}],\n", - " u'id': u'56bec7353aeaaa14008c940d',\n", - " u'question': u'On what date did the NFL announce that Coldplay would headline the halftime show?'},\n", - " {u'answers': [{u'answer_start': 185,\n", - " u'text': u'British'},\n", - " {u'answer_start': 185,\n", - " u'text': u'British'},\n", - " {u'answer_start': 185,\n", - " u'text': u'British'}],\n", - " u'id': u'56bec7353aeaaa14008c940e',\n", - " u'question': u'What nationality is the band Coldplay?'},\n", - " {u'answers': [{u'answer_start': 306,\n", - " u'text': u'Super Bowl XLVII'},\n", - " {u'answer_start': 306,\n", - " u'text': u'Super Bowl XLVII'},\n", - " {u'answer_start': 317,\n", - " u'text': u'XLVII'}],\n", - " u'id': u'56bec7353aeaaa14008c940f',\n", - " u'question': u'At what Super Bowl did Beyonc\\xe9 headline the halftime show?'},\n", - " {u'answers': [{u'answer_start': 382,\n", - " u'text': u'\"Hymn for the Weekend\"'},\n", - " {u'answer_start': 383,\n", - " u'text': u'Hymn for the Weekend'},\n", - " {u'answer_start': 383,\n", - " u'text': u'Hymn for the Weekend'}],\n", - " u'id': u'56bec7353aeaaa14008c9410',\n", - " u'question': u'What was the name of the single that Coldplay and Beyonc\\xe9 collaborated on?'},\n", - " {u'answers': [{u'answer_start': 467,\n", - " u'text': u'Super Bowl XLVIII'},\n", - " {u'answer_start': 467,\n", - " u'text': u'Super Bowl XLVIII'},\n", - " {u'answer_start': 478,\n", - " u'text': u'XLVIII'}],\n", - " u'id': u'56bec7353aeaaa14008c9411',\n", - " u'question': u'What previous Super Bowl halftime show did Bruno Mars headline?'},\n", - " {u'answers': [{u'answer_start': 204,\n", - " u'text': u'Coldplay.'},\n", - " {u'answer_start': 204,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 204,\n", - " u'text': u'Coldplay'}],\n", - " u'id': u'56d725790d65d214001983d8',\n", - " u'question': u'Who lead the halftime show of Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 234,\n", - " u'text': u'Pepsi'},\n", - " {u'answer_start': 234,\n", - " u'text': u'Pepsi'},\n", - " {u'answer_start': 234,\n", - " u'text': u'Pepsi'}],\n", - " u'id': u'56d725790d65d214001983da',\n", - " u'question': u'What company confirmed that Beyonc\\xe9 would participate in the Super Bowl 50 halftime show?'},\n", - " {u'answers': [{u'answer_start': 382,\n", - " u'text': u'\"Hymn for the Weekend\"'},\n", - " {u'answer_start': 383,\n", - " u'text': u'Hymn for the Weekend'},\n", - " {u'answer_start': 383,\n", - " u'text': u'Hymn for the Weekend'}],\n", - " u'id': u'56d725790d65d214001983db',\n", - " u'question': u'What song did Coldplay and Beyonc\\xe9 collaborate on for the Super Bowl 50 halftime show?'},\n", - " {u'answers': [{u'answer_start': 204,\n", - " u'text': u'Coldplay.'},\n", - " {u'answer_start': 204,\n", - " u'text': u'Coldplay'},\n", - " {u'answer_start': 204,\n", - " u'text': u'Coldplay'}],\n", - " u'id': u'56d9c551dc89441400fdb7d0',\n", - " u'question': u'Who headlined the Super Bowl 50 halftime show?'},\n", - " {u'answers': [{u'answer_start': 279,\n", - " u'text': u'Beyonc\\xe9'},\n", - " {u'answer_start': 279,\n", - " u'text': u'Beyonc\\xe9'},\n", - " {u'answer_start': 279,\n", - " u'text': u'Beyonc\\xe9'}],\n", - " u'id': u'56d9c551dc89441400fdb7d1',\n", - " u'question': u'On January 7, 2016, it was confirmed that which start would join Coldplay for the halftime show?'},\n", - " {u'answers': [{u'answer_start': 383,\n", - " u'text': u'Hymn for the Weekend'},\n", - " {u'answer_start': 383,\n", - " u'text': u'Hymn for the Weekend'},\n", - " {u'answer_start': 443,\n", - " u'text': u'Mars'}],\n", - " u'id': u'56d9c551dc89441400fdb7d2',\n", - " u'question': u'Which single did Beyonc\\xe9 and Coldplay collaborate on?'},\n", - " {u'answers': [{u'answer_start': 437,\n", - " u'text': u'Bruno Mars'},\n", - " {u'answer_start': 437,\n", - " u'text': u'Bruno Mars'},\n", - " {u'answer_start': 437,\n", - " u'text': u'Bruno Mars,'}],\n", - " u'id': u'56d9c551dc89441400fdb7d3',\n", - " u'question': u'What other former halftime show performer joined Coldplay and Beyonc\\xe9 for the Super Bowl 50 halftime show?'}]},\n", - " {u'context': u\"Denver took the opening kickoff and started out strong with Peyton Manning completing an 18-yard pass to tight end Owen Daniels and a 22-yard throw to receiver Andre Caldwell. A pair of carries by C. J. Anderson moved the ball up 20 yards to the Panthers 14-yard line, but Carolina's defense dug in over the next three plays. First, linebacker Shaq Thompson tackled Ronnie Hillman for a 3-yard loss. Then after an incompletion, Thomas Davis tackled Anderson for a 1-yard gain on third down, forcing Denver to settle for a 3\\u20130 lead on a Brandon McManus 34-yard field goal. The score marked the first time in the entire postseason that Carolina was facing a deficit.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 0,\n", - " u'text': u'Denver'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Denver'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Denver'}],\n", - " u'id': u'56bec7a63aeaaa14008c9417',\n", - " u'question': u'What team received the opening kickoff?'},\n", - " {u'answers': [{u'answer_start': 160,\n", - " u'text': u'Andre Caldwell'},\n", - " {u'answer_start': 160,\n", - " u'text': u'Andre Caldwell'},\n", - " {u'answer_start': 166,\n", - " u'text': u'Caldwell'}],\n", - " u'id': u'56bec7a63aeaaa14008c9418',\n", - " u'question': u'Who was at the receiving end of a 22-yard pass from Peyton Manning?'},\n", - " {u'answers': [{u'answer_start': 366,\n", - " u'text': u'Ronnie Hillman'},\n", - " {u'answer_start': 366,\n", - " u'text': u'Ronnie Hillman'},\n", - " {u'answer_start': 373,\n", - " u'text': u'Hillman'}],\n", - " u'id': u'56bec7a63aeaaa14008c9419',\n", - " u'question': u'Who lost 3 yards on a tackle by Shaq Thompson?'},\n", - " {u'answers': [{u'answer_start': 536,\n", - " u'text': u'Brandon McManus'},\n", - " {u'answer_start': 536,\n", - " u'text': u'Brandon McManus'},\n", - " {u'answer_start': 544,\n", - " u'text': u'McManus'}],\n", - " u'id': u'56bec7a63aeaaa14008c941a',\n", - " u'question': u'Who kicked a field goal for Denver?'},\n", - " {u'answers': [{u'answer_start': 197,\n", - " u'text': u'C. J. Anderson'},\n", - " {u'answer_start': 197,\n", - " u'text': u'C. J. Anderson'},\n", - " {u'answer_start': 203,\n", - " u'text': u'Anderson'}],\n", - " u'id': u'56bec7a63aeaaa14008c941b',\n", - " u'question': u'Who ran the ball twice for 20 yards on this drive?'},\n", - " {u'answers': [{u'answer_start': 89,\n", - " u'text': u'18'},\n", - " {u'answer_start': 89,\n", - " u'text': u'18'},\n", - " {u'answer_start': 134,\n", - " u'text': u'22-yard'}],\n", - " u'id': u'56d728a10d65d21400198413',\n", - " u'question': u'How many yards was the pass on the first drive?'},\n", - " {u'answers': [{u'answer_start': 344,\n", - " u'text': u'Shaq Thompson'},\n", - " {u'answer_start': 344,\n", - " u'text': u'Shaq Thompson'},\n", - " {u'answer_start': 349,\n", - " u'text': u'Thompson'}],\n", - " u'id': u'56d728a10d65d21400198414',\n", - " u'question': u'Who tackled Ronnie Hillman for a loss of yardage?'},\n", - " {u'answers': [{u'answer_start': 536,\n", - " u'text': u'Brandon McManus'},\n", - " {u'answer_start': 536,\n", - " u'text': u'Brandon McManus'},\n", - " {u'answer_start': 544,\n", - " u'text': u'McManus'}],\n", - " u'id': u'56d728a10d65d21400198415',\n", - " u'question': u'Who kicked the field goal for Denver on the first drive of Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 654,\n", - " u'text': u'a deficit.'},\n", - " {u'answer_start': 654,\n", - " u'text': u'a deficit'},\n", - " {u'answer_start': 656,\n", - " u'text': u'deficit'}],\n", - " u'id': u'56d728a10d65d21400198416',\n", - " u'question': u'This was the first time that the Carolina team faced what in the post season?'},\n", - " {u'answers': [{u'answer_start': 0,\n", - " u'text': u'Denver'},\n", - " {u'answer_start': 634,\n", - " u'text': u'Carolina'},\n", - " {u'answer_start': 0,\n", - " u'text': u'Denver'}],\n", - " u'id': u'56d9c5e7dc89441400fdb7d8',\n", - " u'question': u'Which team had the first kickoff?'},\n", - " {u'answers': [{u'answer_start': 115,\n", - " u'text': u'Owen Daniels'},\n", - " {u'answer_start': 115,\n", - " u'text': u'Owen Daniels'},\n", - " {u'answer_start': 120,\n", - " u'text': u'Daniels'}],\n", - " u'id': u'56d9c5e7dc89441400fdb7d9',\n", - " u'question': u'Who did Peyton Manning complete an 18 yard pass to on the opening drive?'},\n", - " {u'answers': [{u'answer_start': 197,\n", - " u'text': u'C. J. Anderson'},\n", - " {u'answer_start': 197,\n", - " u'text': u'C. J. Anderson'},\n", - " {u'answer_start': 203,\n", - " u'text': u'Anderson'}],\n", - " u'id': u'56d9c5e7dc89441400fdb7da',\n", - " u'question': u'Which Denver player took the ball to the Panthers 14 yard line?'},\n", - " {u'answers': [{u'answer_start': 536,\n", - " u'text': u'Brandon McManus'},\n", - " {u'answer_start': 536,\n", - " u'text': u'Brandon McManus'},\n", - " {u'answer_start': 544,\n", - " u'text': u'McManus'}],\n", - " u'id': u'56d9c5e7dc89441400fdb7db',\n", - " u'question': u'Who scored the first points for Denver?'},\n", - " {u'answers': [{u'answer_start': 654,\n", - " u'text': u'a deficit.'},\n", - " {u'answer_start': 654,\n", - " u'text': u'a deficit'},\n", - " {u'answer_start': 656,\n", - " u'text': u'deficit'}],\n", - " u'id': u'56d9c5e7dc89441400fdb7dc',\n", - " u'question': u'What did Carolina face in the opening drive that they had not faced the entire postseason?'}]},\n", - " {u'context': u\"After each team punted, Panthers quarterback Cam Newton appeared to complete a 24-yard pass Jerricho Cotchery, but the call was ruled an incompletion and upheld after a replay challenge. CBS analyst and retired referee Mike Carey stated he disagreed with the call and felt the review clearly showed the pass was complete. A few plays later, on 3rd-and-10 from the 15-yard line, linebacker Von Miller knocked the ball out of Newton's hands while sacking him, and Malik Jackson recovered it in the end zone for a Broncos touchdown, giving the team a 10\\u20130 lead. This was the first fumble return touchdown in a Super Bowl since Super Bowl XXVIII at the end of the 1993 season.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 219,\n", - " u'text': u'Mike Carey'},\n", - " {u'answer_start': 219,\n", - " u'text': u'Mike Carey'},\n", - " {u'answer_start': 224,\n", - " u'text': u'Carey'}],\n", - " u'id': u'56bec8243aeaaa14008c942b',\n", - " u'question': u'Which former referee served as an analyst for CBS?'},\n", - " {u'answers': [{u'answer_start': 45,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 45,\n", - " u'text': u'Cam Newton'},\n", - " {u'answer_start': 49,\n", - " u'text': u'Newton'}],\n", - " u'id': u'56bec8243aeaaa14008c942c',\n", - " u'question': u'Whose pass to Jerricho Cotchery was ruled incomplete?'},\n", - " {u'answers': [{u'answer_start': 389,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 389,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 393,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56bec8243aeaaa14008c942d',\n", - " u'question': u'Who stripped the ball from Cam Newton while sacking him on this drive?'},\n", - " {u'answers': [{u'answer_start': 462,\n", - " u'text': u'Malik Jackson'},\n", - " {u'answer_start': 462,\n", - " u'text': u'Malik Jackson'},\n", - " {u'answer_start': 468,\n", - " u'text': u'Jackson'}],\n", - " u'id': u'56bec8243aeaaa14008c942e',\n", - " u'question': u'Which player recovered possession of the ball in the end zone?'},\n", - " {u'answers': [{u'answer_start': 624,\n", - " u'text': u'Super Bowl XXVIII'},\n", - " {u'answer_start': 624,\n", - " u'text': u'Super Bowl XXVIII'},\n", - " {u'answer_start': 635,\n", - " u'text': u'XXVIII'}],\n", - " u'id': u'56bec8243aeaaa14008c942f',\n", - " u'question': u'What Super Bowl was the last where a fumble was returned for a touchdown?'},\n", - " {u'answers': [{u'answer_start': 92,\n", - " u'text': u'Jerricho Cotchery'},\n", - " {u'answer_start': 92,\n", - " u'text': u'Jerricho Cotchery'},\n", - " {u'answer_start': 101,\n", - " u'text': u'Cotchery'}],\n", - " u'id': u'56d728e00d65d2140019841c',\n", - " u'question': u'Who received the pass that was ruled incomplete and confirmed after a challenge?'},\n", - " {u'answers': [{u'answer_start': 219,\n", - " u'text': u'Mike Carey'},\n", - " {u'answer_start': 219,\n", - " u'text': u'Mike Carey'},\n", - " {u'answer_start': 224,\n", - " u'text': u'Carey'}],\n", - " u'id': u'56d728e00d65d2140019841d',\n", - " u'question': u'What analyst did not agree with the ruling of the incomplete pass?'},\n", - " {u'answers': [{u'answer_start': 389,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 389,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 393,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56d728e00d65d2140019841e',\n", - " u'question': u'Who, according to the text, caused Cam Newton to lose the ball during a tackle?'},\n", - " {u'answers': [{u'answer_start': 462,\n", - " u'text': u'Malik Jackson'},\n", - " {u'answer_start': 462,\n", - " u'text': u'Malik Jackson'},\n", - " {u'answer_start': 468,\n", - " u'text': u'Jackson'}],\n", - " u'id': u'56d728e00d65d2140019841f',\n", - " u'question': u'Who received the ball that cam Newton lost during a tackle to make it a Denver touchdown?'},\n", - " {u'answers': [{u'answer_start': 660,\n", - " u'text': u'1993'},\n", - " {u'answer_start': 660,\n", - " u'text': u'1993'},\n", - " {u'answer_start': 660,\n", - " u'text': u'1993'}],\n", - " u'id': u'56d728e00d65d21400198420',\n", - " u'question': u'What year was the last time a fumble return touchdown like this occurred?'},\n", - " {u'answers': [{u'answer_start': 219,\n", - " u'text': u'Mike Carey'},\n", - " {u'answer_start': 219,\n", - " u'text': u'Mike Carey'},\n", - " {u'answer_start': 224,\n", - " u'text': u'Carey'}],\n", - " u'id': u'56d9c660dc89441400fdb7e7',\n", - " u'question': u'Which retired ref disagreed with the outcome of the challenge of a incomplete pass?'},\n", - " {u'answers': [{u'answer_start': 389,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 389,\n", - " u'text': u'Von Miller'},\n", - " {u'answer_start': 393,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56d9c660dc89441400fdb7e8',\n", - " u'question': u'Who sacked Newton a few plays after the challenge?'},\n", - " {u'answers': [{u'answer_start': 462,\n", - " u'text': u'Malik Jackson'},\n", - " {u'answer_start': 462,\n", - " u'text': u'Malik Jackson'},\n", - " {u'answer_start': 468,\n", - " u'text': u'Jackson'}],\n", - " u'id': u'56d9c660dc89441400fdb7e9',\n", - " u'question': u'Who recovered the fumble and ran the ball in for a touchdown?'},\n", - " {u'answers': [{u'answer_start': 624,\n", - " u'text': u'Super Bowl XXVIII'},\n", - " {u'answer_start': 645,\n", - " u'text': u'the end of the 1993 season'},\n", - " {u'answer_start': 660,\n", - " u'text': u'1993'}],\n", - " u'id': u'56d9c660dc89441400fdb7ea',\n", - " u'question': u'When is the last time a fumble return touchdown happened in a Super Bowl?'}]},\n", - " {u'context': u\"After a punt from both teams, Carolina got on track with a 9-play, 73-yard scoring drive. Newton completed 4 of 4 passes for 51 yards and rushed twice for 25 yards, while Jonathan Stewart finished the drive with a 1-yard touchdown run, cutting the score to 10\\u20137 with 11:28 left in the second quarter. Later on, Broncos receiver Jordan Norwood received Brad Nortman's short 28-yard punt surrounded by Panthers players, but none of them attempted to make a tackle, apparently thinking Norwood had called a fair catch. Norwood had not done so, and with no resistance around him, he took off for a Super Bowl record 61-yard return before Mario Addison dragged him down on the Panthers 14-yard line. Despite Denver's excellent field position, they could not get the ball into the end zone, so McManus kicked a 33-yard field goal that increased their lead to 13\\u20137.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 171,\n", - " u'text': u'Jonathan Stewart'},\n", - " {u'answer_start': 171,\n", - " u'text': u'Jonathan Stewart'},\n", - " {u'answer_start': 180,\n", - " u'text': u'Stewart'}],\n", - " u'id': u'56bec8a13aeaaa14008c9435',\n", - " u'question': u'Who ran for a Carolina touchdown to make the score 10\\u20137?'},\n", - " {u'answers': [{u'answer_start': 352,\n", - " u'text': u'Brad Nortman'},\n", - " {u'answer_start': 352,\n", - " u'text': u'Brad Nortman'},\n", - " {u'answer_start': 357,\n", - " u'text': u'Nortman'}],\n", - " u'id': u'56bec8a13aeaaa14008c9436',\n", - " u'question': u'Who punted the ball for Carolina?'},\n", - " {u'answers': [{u'answer_start': 270,\n", - " u'text': u'28'},\n", - " {u'answer_start': 373,\n", - " u'text': u'28'},\n", - " {u'answer_start': 373,\n", - " u'text': u'28'}],\n", - " u'id': u'56bec8a13aeaaa14008c9437',\n", - " u'question': u\"How many yards did Brad Nortman's punt travel?\"},\n", - " {u'answers': [{u'answer_start': 612,\n", - " u'text': u'61'},\n", - " {u'answer_start': 612,\n", - " u'text': u'61'},\n", - " {u'answer_start': 612,\n", - " u'text': u'61'}],\n", - " u'id': u'56bec8a13aeaaa14008c9438',\n", - " u'question': u'How many yards did Jordan Norwood return a punt to set the Super Bowl record?'},\n", - " {u'answers': [{u'answer_start': 805,\n", - " u'text': u'33'},\n", - " {u'answer_start': 805,\n", - " u'text': u'33'},\n", - " {u'answer_start': 805,\n", - " u'text': u'33'}],\n", - " u'id': u'56bec8a13aeaaa14008c9439',\n", - " u'question': u'How many yards was the McManus field goal?'},\n", - " {u'answers': [{u'answer_start': 125,\n", - " u'text': u'51'},\n", - " {u'answer_start': 125,\n", - " u'text': u'51'},\n", - " {u'answer_start': 125,\n", - " u'text': u'51'}],\n", - " u'id': u'56d729180d65d21400198426',\n", - " u'question': u'How many passing yards did Cam Newton get for his 4 of 4 passes?'},\n", - " {u'answers': [{u'answer_start': 171,\n", - " u'text': u'Jonathan Stewart'},\n", - " {u'answer_start': 171,\n", - " u'text': u'Jonathan Stewart'},\n", - " {u'answer_start': 180,\n", - " u'text': u'Stewart'}],\n", - " u'id': u'56d729180d65d21400198427',\n", - " u'question': u'Who got a touchdown making the score 10-7?'},\n", - " {u'answers': [{u'answer_start': 267,\n", - " u'text': u'11:28'},\n", - " {u'answer_start': 267,\n", - " u'text': u'11:28'},\n", - " {u'answer_start': 267,\n", - " u'text': u'11:28'}],\n", - " u'id': u'56d729180d65d21400198428',\n", - " u'question': u'How much time was left in the quarter when Stewart got the touchdown?'},\n", - " {u'answers': [{u'answer_start': 328,\n", - " u'text': u'Jordan Norwood'},\n", - " {u'answer_start': 328,\n", - " u'text': u'Jordan Norwood'},\n", - " {u'answer_start': 516,\n", - " u'text': u'Norwood'}],\n", - " u'id': u'56d729180d65d21400198429',\n", - " u'question': u'Who got a 61-yard return when players thought he called for a fair catch but did not?'},\n", - " {u'answers': [{u'answer_start': 805,\n", - " u'text': u'33'},\n", - " {u'answer_start': 805,\n", - " u'text': u'33'},\n", - " {u'answer_start': 805,\n", - " u'text': u'33'}],\n", - " u'id': u'56d729180d65d2140019842a',\n", - " u'question': u'How many yards was the field goal that made the score 13-7 in Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 171,\n", - " u'text': u'Jonathan Stewart'},\n", - " {u'answer_start': 171,\n", - " u'text': u'Jonathan Stewart'},\n", - " {u'answer_start': 180,\n", - " u'text': u'Stewart'}],\n", - " u'id': u'56d9c6e0dc89441400fdb7f0',\n", - " u'question': u'Who scored the Panthers first touchdown? '},\n", - " {u'answers': [{u'answer_start': 813,\n", - " u'text': u'field goal'},\n", - " {u'answer_start': 813,\n", - " u'text': u'field goal'},\n", - " {u'answer_start': 813,\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " u'text': u'field goal'}],\n", - " u'id': u'56d9c6e0dc89441400fdb7f4',\n", - " u'question': u'How did Denver score at the end of the drive?'}]},\n", - " {u'context': u\"On Carolina's next possession fullback Mike Tolbert lost a fumble while being tackled by safety Darian Stewart, which linebacker Danny Trevathan recovered on the Broncos 40-yard line. However, the Panthers soon took the ball back when defensive end Kony Ealy tipped a Manning pass to himself and then intercepted it, returning the ball 19 yards to the Panthers 39-yard line with 1:55 left on the clock. The Panthers could not gain any yards with their possession and had to punt. After a Denver punt, Carolina drove to the Broncos 45-yard line. But with 11 seconds left, Newton was sacked by DeMarcus Ware as time expired in the half.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 96,\n", - " u'text': u'Darian Stewart'},\n", - " {u'answer_start': 96,\n", - " u'text': u'Darian Stewart'},\n", - " {u'answer_start': 103,\n", - " u'text': u'Stewart'}],\n", - " u'id': u'56bec9133aeaaa14008c9445',\n", - " u'question': u'Who tackled Mike Tolbert and caused a fumble?'},\n", - " {u'answers': [{u'answer_start': 118,\n", - " u'text': u'linebacker'},\n", - " {u'answer_start': 118,\n", - " u'text': u'linebacker'},\n", - " {u'answer_start': 118,\n", - " u'text': u'linebacker'}],\n", - " u'id': u'56bec9133aeaaa14008c9446',\n", - " u'question': u'What position does Danny Trevathan play?'},\n", - " {u'answers': [{u'answer_start': 249,\n", - " u'text': u'Kony Ealy'},\n", - " {u'answer_start': 249,\n", - " u'text': u'Kony Ealy'},\n", - " {u'answer_start': 254,\n", - " u'text': u'Ealy'}],\n", - " u'id': u'56bec9133aeaaa14008c9447',\n", - " u'question': u'Which Panther tipped a Manning pass to himself and picked it off?'},\n", - " {u'answers': [{u'answer_start': 571,\n", - " u'text': u'Newton'},\n", - " {u'answer_start': 571,\n", - " u'text': u'Newton'},\n", - " {u'answer_start': 571,\n", - " u'text': u'Newton'}],\n", - " u'id': u'56bec9133aeaaa14008c9448',\n", - " u'question': u'Who was sacked as the first half clock expired?'},\n", - " {u'answers': [{u'answer_start': 592,\n", - " u'text': u'DeMarcus Ware'},\n", - " {u'answer_start': 592,\n", - " u'text': u'DeMarcus Ware'},\n", - " {u'answer_start': 601,\n", - " u'text': u'Ware'}],\n", - " u'id': u'56bec9133aeaaa14008c9449',\n", - " u'question': u'Who sacked Cam Newton at the end of the half?'},\n", - " {u'answers': [{u'answer_start': 39,\n", - " u'text': u'Mike Tolbert'},\n", - " {u'answer_start': 39,\n", - " u'text': u'Mike Tolbert'},\n", - " {u'answer_start': 44,\n", - " u'text': u'Tolbert'}],\n", - " u'id': u'56d729ec0d65d21400198430',\n", - " u'question': u'Who fumbled the ball when tackled by Darian Stewart?'},\n", - " {u'answers': [{u'answer_start': 249,\n", - " u'text': u'Kony Ealy'},\n", - " {u'answer_start': 249,\n", - " u'text': u'Kony Ealy'},\n", - " {u'answer_start': 254,\n", - " u'text': u'Ealy'}],\n", - " u'id': u'56d729ec0d65d21400198431',\n", - " u'question': u\"Who tipped Peyton's pass for an interception?\"},\n", - " {u'answers': [{u'answer_start': 336,\n", - " u'text': u'19'},\n", - " {u'answer_start': 336,\n", - " u'text': u'19'},\n", - " {u'answer_start': 336,\n", - " u'text': u'19'}],\n", - " u'id': u'56d729ec0d65d21400198432',\n", - " u'question': u'How many yards did the interception gain when the ball was tipped away from Manning?'},\n", - " {u'answers': [{u'answer_start': 592,\n", - " u'text': u'DeMarcus Ware'},\n", - " {u'answer_start': 592,\n", - " u'text': u'DeMarcus Ware'},\n", - " {u'answer_start': 601,\n", - " u'text': u'Ware'}],\n", - " u'id': u'56d729ec0d65d21400198434',\n", - " u'question': u\"Who tackled the Carolina Panthers' quarterback just before the end of the first half?\"},\n", - " {u'answers': [{u'answer_start': 39,\n", - " u'text': u'Mike Tolbert'},\n", - " {u'answer_start': 39,\n", - " u'text': u'Mike Tolbert'},\n", - " {u'answer_start': 44,\n", - " u'text': u'Tolbert'}],\n", - " u'id': u'56d9c79edc89441400fdb804',\n", - " u'question': u'Which fullback fumbled the ball after a Darian Stewart tackle?'},\n", - " {u'answers': [{u'answer_start': 129,\n", - " u'text': u'Danny Trevathan'},\n", - " {u'answer_start': 129,\n", - " u'text': u'Danny Trevathan'},\n", - " {u'answer_start': 103,\n", - " u'text': u'Stewart'}],\n", - " u'id': u'56d9c79edc89441400fdb805',\n", - " u'question': u\"Who recovered Tolbert's fumble?\"},\n", - " {u'answers': [{u'answer_start': 249,\n", - " u'text': u'Kony Ealy'},\n", - " {u'answer_start': 249,\n", - " u'text': u'Kony Ealy'},\n", - " {u'answer_start': 254,\n", - " u'text': u'Ealy'}],\n", - " u'id': u'56d9c79edc89441400fdb806',\n", - " u'question': u'Who intercepted a Manning pass, giving the ball back to the Panthers?'},\n", - " {u'answers': [{u'answer_start': 474,\n", - " u'text': u'punt'},\n", - " {u'answer_start': 571,\n", - " u'text': u'Newton was sacked'},\n", - " {u'answer_start': 582,\n", - " u'text': u'sacked'}],\n", - " u'id': u'56d9c79edc89441400fdb807',\n", - " u'question': u'How did the drive end for the Panthers?'},\n", - " {u'answers': [{u'answer_start': 592,\n", - " u'text': u'DeMarcus Ware'},\n", - " {u'answer_start': 592,\n", - " u'text': u'DeMarcus Ware'},\n", - " {u'answer_start': 601,\n", - " u'text': u'Ware'}],\n", - " u'id': u'56d9c79edc89441400fdb808',\n", - " u'question': u'Who sacked Newton with 11 seconds left in the first half?'}]},\n", - " {u'context': u\"The Panthers seemed primed to score on their opening drive of the second half when Newton completed a 45-yard pass to Ted Ginn Jr. on the Denver 35-yard line on their second offensive play. But the Broncos defense halted the drive on the 26-yard line, and it ended with no points when Graham Gano hit the uprights on a 44-yard field goal attempt. After the miss, Manning completed a pair of passes to Emmanuel Sanders for gains of 25 and 22 yards, setting up McManus' 33-yard field goal that gave the Broncos a 16\\u20137 lead. Carolina got off to another strong start after the kickoff, with Newton completing a 42-yard pass to Corey Brown. But once again they came up empty, this time as a result of a Newton pass that bounced off the hands of Ginn and was intercepted by safety T. J. Ward. Ward fumbled the ball during the return, but Trevathan recovered it to enable Denver to keep possession.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 118,\n", - " u'text': u'Ted Ginn Jr.'},\n", - " {u'answer_start': 118,\n", - " u'text': u'Ted Ginn Jr'},\n", - " {u'answer_start': 118,\n", - " u'text': u'Ted Ginn Jr'}],\n", - " u'id': u'56bec98e3aeaaa14008c9455',\n", - " u'question': u'Who was on the receiving end of a 45-yard pass from Cam Newton?'},\n", - " {u'answers': [{u'answer_start': 285,\n", - " u'text': u'Graham Gano'},\n", - " {u'answer_start': 285,\n", - " u'text': u'Graham Gano'},\n", - " {u'answer_start': 292,\n", - " u'text': u'Gano'}],\n", - " u'id': u'56bec98e3aeaaa14008c9456',\n", - " u'question': u'Who missed a field goal for the Panthers?'},\n", - " {u'answers': [{u'answer_start': 319,\n", - " u'text': u'44'},\n", - " {u'answer_start': 319,\n", - " u'text': u'44'},\n", - " {u'answer_start': 319,\n", - " u'text': u'44'}],\n", - " u'id': u'56bec98e3aeaaa14008c9457',\n", - " u'question': u'How many yards was the missed field goal?'},\n", - " {u'answers': [{u'answer_start': 459,\n", - " u'text': u'McManus'},\n", - " {u'answer_start': 459,\n", - " u'text': u'McManus'},\n", - " {u'answer_start': 459,\n", - " u'text': u'McManus'}],\n", - " u'id': u'56bec98e3aeaaa14008c9458',\n", - " u'question': u'What kicker was successful on a 33-yard field goal?'},\n", - " {u'answers': [{u'answer_start': 775,\n", - " u'text': u'T. J. Ward'},\n", - " {u'answer_start': 775,\n", - " u'text': u'T. J. Ward'},\n", - " {u'answer_start': 781,\n", - " u'text': u'Ward'}],\n", - " u'id': u'56bec98e3aeaaa14008c9459',\n", - " u'question': u'Who picked off Cam Newton and subsequently fumbled the ball?'},\n", - " {u'answers': [{u'answer_start': 118,\n", - " u'text': u'Ted Ginn Jr.'},\n", - " {u'answer_start': 118,\n", - " u'text': u'Ted Ginn Jr.'},\n", - " {u'answer_start': 118,\n", - " u'text': u'Ted Ginn Jr'}],\n", - " u'id': u'56d7282f0d65d21400198408',\n", - " u'question': u'Who did Newton complete a 45 yard pass to on the opening drive of the second half?'},\n", - " {u'answers': [{u'answer_start': 301,\n", - " u'text': u'the uprights'},\n", - " {u'answer_start': 301,\n", - " u'text': u'the uprights'},\n", - " {u'answer_start': 305,\n", - " u'text': u'uprights'}],\n", - " u'id': u'56d7282f0d65d21400198409',\n", - " u'question': u'What did Gano hit on his 44-yard field goal attempt?'},\n", - " {u'answers': [{u'answer_start': 775,\n", - " u'text': u'T. J. Ward.'},\n", - " {u'answer_start': 775,\n", - " u'text': u'T. J. Ward'},\n", - " {u'answer_start': 781,\n", - " u'text': u'Ward'}],\n", - " u'id': u'56d7282f0d65d2140019840a',\n", - " u'question': u\"Which Broncos player intercepted Newton on the Panthers' next set of downs?\"},\n", - " {u'answers': [{u'answer_start': 832,\n", - " u'text': u'Trevathan'},\n", - " {u'answer_start': 832,\n", - " u'text': u'Trevathan'},\n", - " {u'answer_start': 832,\n", - " u'text': u'Trevathan'}],\n", - " u'id': u'56d7282f0d65d2140019840c',\n", - " u'question': u\"Who recovered Ward's fumble?\"},\n", - " {u'answers': [{u'answer_start': 118,\n", - " u'text': u'Ted Ginn Jr.'},\n", - " {u'answer_start': 118,\n", - " u'text': u'Ted Ginn Jr'}],\n", - " u'id': u'56d9c92bdc89441400fdb80e',\n", - " u'question': u'Who did Newton get a pass to in the Panther starting plays of Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 238,\n", - " u'text': u'26-yard line'},\n", - " {u'answer_start': 238,\n", - " u'text': u'26'}],\n", - " u'id': u'56d9c92bdc89441400fdb80f',\n", - " u'question': u'What yard line did the Broncos stop the Panthers on in their first offensive drive of Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 285,\n", - " u'text': u'Graham Gano'},\n", - " {u'answer_start': 285,\n", - " u'text': u'Graham Gano'},\n", - " {u'answer_start': 292,\n", - " u'text': u'Gano'}],\n", - " u'id': u'56d9c92bdc89441400fdb810',\n", - " u'question': u'Who hit the goal post on a field goal attempt?'},\n", - " {u'answers': [{u'answer_start': 401,\n", - " u'text': u'Emmanuel Sanders'},\n", - " {u'answer_start': 401,\n", - " u'text': u'Emmanuel Sanders'},\n", - " {u'answer_start': 410,\n", - " u'text': u'Sanders'}],\n", - " u'id': u'56d9c92bdc89441400fdb811',\n", - " u'question': u'Who caught two passes from Manning after the failed Carolina field goal attempt?'}]},\n", - " {u'context': u\"There would be no more scoring in the third quarter, but early in the fourth, the Broncos drove to the Panthers 41-yard line. On the next play, Ealy knocked the ball out of Manning's hand as he was winding up for a pass, and then recovered it for Carolina on the 50-yard line. A 16-yard reception by Devin Funchess and a 12-yard run by Stewart then set up Gano's 39-yard field goal, cutting the Panthers deficit to one score at 16\\u201310. The next three drives of the game would end in punts.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 144,\n", - " u'text': u'Ealy'},\n", - " {u'answer_start': 144,\n", - " u'text': u'Ealy'},\n", - " {u'answer_start': 144,\n", - " u'text': u'Ealy'}],\n", - " u'id': u'56bec9e83aeaaa14008c945f',\n", - " u'question': u'Who recovered a Manning fumble?'},\n", - " {u'answers': [{u'answer_start': 363,\n", - " u'text': u'39'},\n", - " {u'answer_start': 363,\n", - " u'text': u'39'},\n", - " {u'answer_start': 363,\n", - " u'text': u'39'}],\n", - " u'id': u'56bec9e83aeaaa14008c9460',\n", - " u'question': u'Graham Gano kicked a field goal of how many yards?'},\n", - " {u'answers': [{u'answer_start': 300,\n", - " u'text': u'Devin Funchess'},\n", - " {u'answer_start': 300,\n", - " u'text': u'Devin Funchess'},\n", - " {u'answer_start': 306,\n", - " u'text': u'Funchess'}],\n", - " u'id': u'56bec9e83aeaaa14008c9461',\n", - " u'question': u'Who caught a 16-yard pass on this drive?'},\n", - " {u'answers': [{u'answer_start': 336,\n", - " u'text': u'Stewart'},\n", - " {u'answer_start': 336,\n", - " u'text': u'Stewart'},\n", - " {u'answer_start': 336,\n", - " u'text': u'Stewart'}],\n", - " u'id': u'56bec9e83aeaaa14008c9462',\n", - " u'question': u'Who had a 12-yard rush on this drive?'},\n", - " {u'answers': [{u'answer_start': 112,\n", - " u'text': u'41-yard line.'},\n", - " {u'answer_start': 112,\n", - " u'text': u'41'},\n", - " {u'answer_start': 112,\n", - " u'text': u'41'}],\n", - " u'id': u'56d7261e0d65d214001983e0',\n", - " u'question': u'Early in the fourth quarter, The Broncos drove to what yard line of the Panthers?'},\n", - " {u'answers': [{u'answer_start': 144,\n", - " u'text': u'Ealy'},\n", - " {u'answer_start': 144,\n", - " u'text': u'Ealy'},\n", - " {u'answer_start': 144,\n", - " u'text': u'Ealy'}],\n", - " u'id': u'56d7261e0d65d214001983e1',\n", - " u'question': u\"Who knocked the ball out of Manning's hands on the next play?\"},\n", - " {u'answers': [{u'answer_start': 263,\n", - " u'text': u'50-yard line.'},\n", - " {u'answer_start': 256,\n", - " u'text': u'on the 50-yard line'},\n", - " {u'answer_start': 263,\n", - " u'text': u'50'}],\n", - " u'id': u'56d7261e0d65d214001983e2',\n", - " u'question': u'Where was the ball recovered?'},\n", - " {u'answers': [{u'answer_start': 482,\n", - " u'text': u'punts.'},\n", - " {u'answer_start': 482,\n", - " u'text': u'punts'},\n", - " {u'answer_start': 482,\n", - " u'text': u'punts'}],\n", - " u'id': u'56d7261e0d65d214001983e4',\n", - " u'question': u'What did the next three drives result in?'},\n", - " {u'answers': [{u'answer_start': 144,\n", - " u'text': u'Ealy'},\n", - " {u'answer_start': 144,\n", - " u'text': u'Ealy'},\n", - " {u'answer_start': 144,\n", - " u'text': u'Ealy'}],\n", - " u'id': u'56d9ca0adc89441400fdb820',\n", - " u'question': u'What is the last name of the player who bumped the ball away from Manning in the fourth quarter of Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 263,\n", - " u'text': u'50-yard line.'},\n", - " {u'answer_start': 112,\n", - " u'text': u'41'},\n", - " {u'answer_start': 263,\n", - " u'text': u'50'}],\n", - " u'id': u'56d9ca0adc89441400fdb821',\n", - " u'question': u'What yard line was the Broncos on when Manning lost the ball in the fourth quarter?'},\n", - " {u'answers': [{u'answer_start': 363,\n", - " u'text': u'39-yard'},\n", - " {u'answer_start': 363,\n", - " u'text': u'39'},\n", - " {u'answer_start': 363,\n", - " u'text': u'39'}],\n", - " u'id': u'56d9ca0adc89441400fdb822',\n", - " u'question': u'How many yards was the field goal Gano made to make the score 16-10?'},\n", - " {u'answers': [{u'answer_start': 444,\n", - " u'text': u'three'},\n", - " {u'answer_start': 444,\n", - " u'text': u'three'},\n", - " {u'answer_start': 435,\n", - " u'text': u'The next three drives'}],\n", - " u'id': u'56d9ca0adc89441400fdb823',\n", - " u'question': u'How many of the following three fourth quarter drives after the field goal makng the score 16-10 ended in punts?'}]},\n", - " {u'context': u\"With 4:51 left in regulation, Carolina got the ball on their own 24-yard line with a chance to mount a game-winning drive, and soon faced 3rd-and-9. On the next play, Miller stripped the ball away from Newton, and after several players dove for it, it took a long bounce backwards and was recovered by Ward, who returned it five yards to the Panthers 4-yard line. Although several players dove into the pile to attempt to recover it, Newton did not and his lack of aggression later earned him heavy criticism. Meanwhile, Denver's offense was kept out of the end zone for three plays, but a holding penalty on cornerback Josh Norman gave the Broncos a new set of downs. Then Anderson scored on a 2-yard touchdown run and Manning completed a pass to Bennie Fowler for a 2-point conversion, giving Denver a 24\\u201310 lead with 3:08 left and essentially putting the game away. Carolina had two more drives, but failed to get a first down on each one.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 65,\n", - " u'text': u'24'},\n", - " {u'answer_start': 55,\n", - " u'text': u'their own 24'},\n", - " {u'answer_start': 65,\n", - " u'text': u'24'}],\n", - " u'id': u'56beca913aeaaa14008c946d',\n", - " u'question': u'On what yard line did Carolina begin with 4:51 left in the game?'},\n", - " {u'answers': [{u'answer_start': 202,\n", - " u'text': u'Newton'},\n", - " {u'answer_start': 202,\n", - " u'text': u'Newton'},\n", - " {u'answer_start': 434,\n", - " u'text': u'Newton'}],\n", - " u'id': u'56beca913aeaaa14008c946e',\n", - " u'question': u'Who fumbled the ball on 3rd-and-9?'},\n", - " {u'answers': [{u'answer_start': 620,\n", - " u'text': u'Josh Norman'},\n", - " {u'answer_start': 620,\n", - " u'text': u'Josh Norman'},\n", - " {u'answer_start': 625,\n", - " u'text': u'Norman'}],\n", - " u'id': u'56beca913aeaaa14008c946f',\n", - " u'question': u'What Panther defender was called for holding on third down?'},\n", - " {u'answers': [{u'answer_start': 674,\n", - " u'text': u'Anderson'},\n", - " {u'answer_start': 674,\n", - " u'text': u'Anderson'},\n", - " {u'answer_start': 674,\n", - " u'text': u'Anderson'}],\n", - " u'id': u'56beca913aeaaa14008c9470',\n", - " u'question': u'What player punched the ball in from the 2?'},\n", - " {u'answers': [{u'answer_start': 748,\n", - " u'text': u'Bennie Fowler'},\n", - " {u'answer_start': 748,\n", - " u'text': u'Bennie Fowler'},\n", - " {u'answer_start': 755,\n", - " u'text': u'Fowler'}],\n", - " u'id': u'56beca913aeaaa14008c9471',\n", - " u'question': u'Who was the receiver on the successful 2-point conversion?'},\n", - " {u'answers': [{u'answer_start': 167,\n", - " u'text': u'Miller'},\n", - " {u'answer_start': 167,\n", - " u'text': u'Miller'},\n", - " {u'answer_start': 167,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56d726b60d65d214001983ea',\n", - " u'question': u'Who stripped the ball from Newton on a 3rd and nine?'},\n", - " {u'answers': [{u'answer_start': 275,\n", - " u'text': u'wards'},\n", - " {u'answer_start': 302,\n", - " u'text': u'Ward'},\n", - " {u'answer_start': 302,\n", - " u'text': u'Ward'}],\n", - " u'id': u'56d726b60d65d214001983eb',\n", - " u'question': u'Who recovered the strip ball?'},\n", - " {u'answers': [{u'answer_start': 202,\n", - " u'text': u'Newton'},\n", - " {u'answer_start': 434,\n", - " u'text': u'Newton'},\n", - " {u'answer_start': 434,\n", - " u'text': u'Newton'}],\n", - " u'id': u'56d726b60d65d214001983ec',\n", - " u'question': u'Which player was criticized for not jumping into the pile to recover the ball?'},\n", - " {u'answers': [{u'answer_start': 620,\n", - " u'text': u'Josh Norman'},\n", - " {u'answer_start': 620,\n", - " u'text': u'Josh Norman'},\n", - " {u'answer_start': 625,\n", - " u'text': u'Norman'}],\n", - " u'id': u'56d726b60d65d214001983ed',\n", - " u'question': u'Which Panthers player got a penalty, which gave the Broncos a new set of downs?'},\n", - " {u'answers': [{u'answer_start': 820,\n", - " u'text': u'3:08'},\n", - " {u'answer_start': 820,\n", - " u'text': u'3:08'},\n", - " {u'answer_start': 820,\n", - " u'text': u'3:08'}],\n", - " u'id': u'56d726b60d65d214001983ee',\n", - " u'question': u'How much time was left in the game when Denver took the score to 24-10?'},\n", - " {u'answers': [{u'answer_start': 5,\n", - " u'text': u'4:51'},\n", - " {u'answer_start': 5,\n", - " u'text': u'4:51'},\n", - " {u'answer_start': 5,\n", - " u'text': u'4:51'}],\n", - " u'id': u'56d9cb47dc89441400fdb832',\n", - " u'question': u'What was the time on the clock when Carolina got the ball to their 24-yard line in the fourth quarter?'},\n", - " {u'answers': [{u'answer_start': 167,\n", - " u'text': u'Miller'},\n", - " {u'answer_start': 167,\n", - " u'text': u'Miller'},\n", - " {u'answer_start': 167,\n", - " u'text': u'Miller'}],\n", - " u'id': u'56d9cb47dc89441400fdb833',\n", - " u'question': u'What was the last name of the player who took the ball away from Newton late in the fourth quarter?'},\n", - " {u'answers': [{u'answer_start': 275,\n", - " u'text': u'wards'},\n", - " {u'answer_start': 302,\n", - " u'text': u'Ward'},\n", - " {u'answer_start': 302,\n", - " u'text': u'Ward'}],\n", - " u'id': u'56d9cb47dc89441400fdb834',\n", - " u'question': u'What is the last name of the player who finally recovered the ball Newton lost in the fourth quarter?'},\n", - " {u'answers': [{u'answer_start': 571,\n", - " u'text': u'three'},\n", - " {u'answer_start': 571,\n", - " u'text': u'three'},\n", - " {u'answer_start': 571,\n", - " u'text': u'three'}],\n", - " u'id': u'56d9cb47dc89441400fdb835',\n", - " u'question': u'How many plays was Denver kept out of the end zone after getting the ball from Newton?'},\n", - " {u'answers': [{u'answer_start': 674,\n", - " u'text': u'Anderson'},\n", - " {u'answer_start': 674,\n", - " u'text': u'Anderson'},\n", - " {u'answer_start': 674,\n", - " u'text': u'Anderson'}],\n", - " u'id': u'56d9cb47dc89441400fdb836',\n", - " u'question': u'What is the last name of the player who scored a 2-yard touchdown in the fourth quarter?'}]},\n", - " {u'context': u\"Manning finished the game 13 of 23 for 141 yards with one interception and zero touchdowns. Sanders was his top receiver with six receptions for 83 yards. Anderson was the game's leading rusher with 90 yards and a touchdown, along with four receptions for 10 yards. Miller had six total tackles (five solo), 2\\xbd sacks, and two forced fumbles. Ware had five total tackles and two sacks. Ward had seven total tackles, a fumble recovery, and an interception. McManus made all four of his field goals, making him perfect on all 11 attempts during the post-season. Newton completed 18 of 41 passes for 265 yards, with one interception. He was also the team's leading rusher with 45 yards on six carries. Brown caught four passes for 80 yards, while Ginn had four receptions for 74. Ealy was the top defensive performer for Carolina with four total tackles, three sacks, a forced fumble, a fumble recovery, and an interception. Defensive End Charles Johnson had four total tackles, a sack, and a forced fumble. Linebacker Luke Kuechly had 11 total tackles, while Thomas Davis had seven, despite playing just two weeks after breaking his right arm in the NFC title game.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 296,\n", - " u'text': u'five'},\n", - " {u'answer_start': 308,\n", - " u'text': u'2\\xbd'},\n", - " {u'answer_start': 308,\n", - " u'text': u'2'}],\n", - " u'id': u'56becaf93aeaaa14008c9481',\n", - " u'question': u'How many solo sacks did Von Miller have in the game?'},\n", - " {u'answers': [{u'answer_start': 75,\n", - " u'text': u'zero'},\n", - " {u'answer_start': 75,\n", - " u'text': u'zero'},\n", - " {u'answer_start': 75,\n", - " u'text': u'zero'}],\n", - " u'id': u'56becaf93aeaaa14008c9482',\n", - " u'question': u'How many touchdowns did Manning throw in the game?'},\n", - " {u'answers': [{u'answer_start': 236,\n", - " u'text': u'four'},\n", - " {u'answer_start': 472,\n", - " u'text': u'four'},\n", - " {u'answer_start': 472,\n", - " u'text': u'four'}],\n", - " u'id': u'56becaf93aeaaa14008c9483',\n", - " u'question': u'How many field goals did McManus kick in the game?'},\n", - " {u'answers': [{u'answer_start': 1056,\n", - " u'text': u'Thomas Davis'},\n", - " {u'answer_start': 1056,\n", - " u'text': u'Thomas Davis'},\n", - " {u'answer_start': 1063,\n", - " u'text': u'Davis'}],\n", - " u'id': u'56becaf93aeaaa14008c9484',\n", - " u'question': u'What player played in the Super Bowl after breaking his arm two weeks before?'},\n", - " {u'answers': [{u'answer_start': 54,\n", - " u'text': u'one'},\n", - " {u'answer_start': 612,\n", - " u'text': u'one'},\n", - " {u'answer_start': 612,\n", - " u'text': u'one'}],\n", - " u'id': u'56becaf93aeaaa14008c9485',\n", - " u'question': u'How many picks did Cam Newton throw?'},\n", - " {u'answers': [{u'answer_start': 54,\n", - " u'text': u'one'},\n", - " {u'answer_start': 54,\n", - " u'text': u'one'},\n", - " {u'answer_start': 54,\n", - " u'text': u'one'}],\n", - " u'id': u'56d7270c0d65d214001983f4',\n", - " u'question': u'How many interceptions did Manning have at the end of the game?'},\n", - " {u'answers': [{u'answer_start': 75,\n", - " u'text': u'zero'},\n", - " {u'answer_start': 75,\n", - " u'text': u'zero'},\n", - " {u'answer_start': 75,\n", - " u'text': u'zero'}],\n", - " u'id': u'56d7270c0d65d214001983f5',\n", - " u'question': u'How many touchdowns did Manning have at the end of the game?'},\n", - " {u'answers': [{u'answer_start': 155,\n", - " u'text': u'Anderson'},\n", - " {u'answer_start': 155,\n", - " u'text': u'Anderson'},\n", - " {u'answer_start': 155,\n", - " u'text': u'Anderson'}],\n", - " u'id': u'56d7270c0d65d214001983f6',\n", - " u'question': u\"Who was the game's leading rusher?\"},\n", - " {u'answers': [{u'answer_start': 92,\n", - " u'text': u'Sanders'},\n", - " {u'answer_start': 92,\n", - " u'text': u'Sanders'},\n", - " {u'answer_start': 92,\n", - " u'text': u'Sanders'}],\n", - " u'id': u'56d7270c0d65d214001983f7',\n", - " u'question': u\"Who was the game's top receiver?\"},\n", - " {u'answers': [{u'answer_start': 1056,\n", - " u'text': u'Thomas Davis'},\n", - " {u'answer_start': 1056,\n", - " u'text': u'Thomas Davis'},\n", - " {u'answer_start': 1063,\n", - " u'text': u'Davis'}],\n", - " u'id': u'56d7270c0d65d214001983f8',\n", - " u'question': u'Which player played even though he broke his arm two weeks prior to the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 92,\n", - " u'text': u'Sanders'},\n", - " {u'answer_start': 92,\n", - " u'text': u'Sanders'},\n", - " {u'answer_start': 92,\n", - " u'text': u'Sanders'}],\n", - " u'id': u'56d9ccacdc89441400fdb842',\n", - " u'question': u\"What is the last name of the player who was Manning's top receiver for Super Bowl 50?\"},\n", - " {u'answers': [{u'answer_start': 155,\n", - " u'text': u'Anderson'},\n", - " {u'answer_start': 155,\n", - " u'text': u'Anderson'},\n", - " {u'answer_start': 155,\n", - " u'text': u'Anderson'}],\n", - " u'id': u'56d9ccacdc89441400fdb843',\n", - " u'question': u\"What is th elast name of the player who was the Super Bowl 50 winner's leading rusher?\"},\n", - " {u'answers': [{u'answer_start': 468,\n", - " u'text': u'all four'},\n", - " {u'answer_start': 468,\n", - " u'text': u'all four'},\n", - " {u'answer_start': 472,\n", - " u'text': u'four'}],\n", - " u'id': u'56d9ccacdc89441400fdb844',\n", - " u'question': u'How many of the four field goal attempts did McManus succeed at during SUper Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 54,\n", - " u'text': u'one'},\n", - " {u'answer_start': 612,\n", - " u'text': u'one'},\n", - " {u'answer_start': 612,\n", - " u'text': u'one'}],\n", - " u'id': u'56d9ccacdc89441400fdb845',\n", - " u'question': u'How many intercpetions did Newton have in Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 236,\n", - " u'text': u'four'},\n", - " {u'answer_start': 955,\n", - " u'text': u'four'},\n", - " {u'answer_start': 955,\n", - " u'text': u'four'}],\n", - " u'id': u'56d9ccacdc89441400fdb846',\n", - " u'question': u'How many total tackles did Charles Johnson have in Super Bowl 50?'}]},\n", - " {u'context': u\"Super Bowl 50 featured numerous records from individuals and teams. Denver won despite being massively outgained in total yards (315 to 194) and first downs (21 to 11). Their 194 yards and 11 first downs were both the lowest totals ever by a Super Bowl winning team. The previous record was 244 yards by the Baltimore Ravens in Super Bowl XXXV. Only seven other teams had ever gained less than 200 yards in a Super Bowl, and all of them had lost. The Broncos' seven sacks tied a Super Bowl record set by the Chicago Bears in Super Bowl XX. Kony Ealy tied a Super Bowl record with three sacks. Jordan Norwood's 61-yard punt return set a new record, surpassing the old record of 45 yards set by John Taylor in Super Bowl XXIII. Denver was just 1-of-14 on third down, while Carolina was barely better at 3-of-15. The two teams' combined third down conversion percentage of 13.8 was a Super Bowl low. Manning and Newton had quarterback passer ratings of 56.6 and 55.4, respectively, and their added total of 112 is a record lowest aggregate passer rating for a Super Bowl. Manning became the oldest quarterback ever to win a Super Bowl at age 39, and the first quarterback ever to win a Super Bowl with two different teams, while Gary Kubiak became the first head coach to win a Super Bowl with the same franchise he went to the Super Bowl with as a player.\",\n", - " u'qas': [{u'answers': [{u'answer_start': 136,\n", - " u'text': u'194'},\n", - " {u'answer_start': 136,\n", - " u'text': u'194'},\n", - " {u'answer_start': 136,\n", - " u'text': u'194'}],\n", - " u'id': u'56becb823aeaaa14008c948b',\n", - " u'question': u'How many total yards did Denver gain?'},\n", - " {u'answers': [{u'answer_start': 164,\n", - " u'text': u'11'},\n", - " {u'answer_start': 164,\n", - " u'text': u'11'},\n", - " {u'answer_start': 164,\n", - " u'text': u'11'}],\n", - " u'id': u'56becb823aeaaa14008c948c',\n", - " u'question': u'How many first downs did Denver have?'},\n", - " {u'answers': [{u'answer_start': 308,\n", - " u'text': u'Baltimore Ravens'},\n", - " {u'answer_start': 304,\n", - " u'text': u'the Baltimore Ravens'},\n", - " {u'answer_start': 318,\n", - " u'text': u'Ravens'}],\n", - " u'id': u'56becb823aeaaa14008c948d',\n", - " u'question': u'What team had 244 yards in Super Bowl XXXV?'},\n", - " {u'answers': [{u'answer_start': 593,\n", - " u'text': u'Jordan Norwood'},\n", - " {u'answer_start': 593,\n", - " u'text': u'Jordan Norwood'},\n", - " {u'answer_start': 600,\n", - " u'text': u'Norwood'}],\n", - " u'id': u'56becb823aeaaa14008c948e',\n", - " u'question': u'Who returned a punt for 61 yards, setting a new Super Bowl record?'},\n", - " {u'answers': [{u'answer_start': 897,\n", - " u'text': u'Manning'},\n", - " {u'answer_start': 1069,\n", - " u'text': u'Manning'},\n", - " {u'answer_start': 1069,\n", - " u'text': u'Manning'}],\n", - " u'id': u'56becb823aeaaa14008c948f',\n", - " u'question': u'Who is the only quarterback to win a Super Bowl with two teams?'},\n", - " {u'answers': [{u'answer_start': 136,\n", - " u'text': u'194'},\n", - " {u'answer_start': 136,\n", - " u'text': u'194'},\n", - " {u'answer_start': 136,\n", - " u'text': u'194'}],\n", - " u'id': u'56d7277c0d65d214001983fe',\n", - " u'question': u'How many yards did Denver have for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 164,\n", - " u'text': u'11'},\n", - " {u'answer_start': 164,\n", - " u'text': u'11'},\n", - " {u'answer_start': 164,\n", - " u'text': u'11'}],\n", - " u'id': u'56d7277c0d65d214001983ff',\n", - " u'question': u'How many first downs did Denver have for Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 508,\n", - " u'text': u'Chicago Bears'},\n", - " {u'answer_start': 504,\n", - " u'text': u'the Chicago Bears'},\n", - " {u'answer_start': 516,\n", - " u'text': u'Bears'}],\n", - " u'id': u'56d7277c0d65d21400198401',\n", - " u'question': u'Who did the Broncos tie with the most sacks in a Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 451,\n", - " u'text': u'Broncos'},\n", - " {u'answer_start': 726,\n", - " u'text': u'Denver'},\n", - " {u'answer_start': 68,\n", - " u'text': u'Denver'}],\n", - " u'id': u'56d7277c0d65d21400198402',\n", - " u'question': u'Which team did Gary Kubiak play for when he was in the Super Bowl?'},\n", - " {u'answers': [{u'answer_start': 158,\n", - " u'text': u'21'},\n", - " {u'answer_start': 158,\n", - " u'text': u'21'},\n", - " {u'answer_start': 158,\n", - " u'text': u'21'}],\n", - " u'id': u'56d9cd9adc89441400fdb84c',\n", - " u'question': u'How many first downs did the Panthers have in Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 164,\n", - " u'text': u'11'},\n", - " {u'answer_start': 164,\n", - " u'text': u'11'},\n", - " {u'answer_start': 164,\n", - " u'text': u'11'}],\n", - " u'id': u'56d9cd9adc89441400fdb84d',\n", - " u'question': u'How many first downs did the Broncos have in Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 447,\n", - " u'text': u'The Broncos'},\n", - " {u'answer_start': 68,\n", - " u'text': u'Denver'},\n", - " {u'answer_start': 68,\n", - " u'text': u'Denver'}],\n", - " u'id': u'56d9cd9adc89441400fdb84e',\n", - " u'question': u'What team had the lowest downs and yards ever in the Super Bowl as of Super Bowl 50?'},\n", - " {u'answers': [{u'answer_start': 508,\n", - " u'text': u'Chicago Bears'},\n", - " {u'answer_start': 504,\n", - " u'text': u'the Chicago Bears'},\n", - " {u'answer_start': 516,\n", - " u'text': u'Bears'}],\n", - " u'id': u'56d9cd9adc89441400fdb84f',\n", - " u'question': u'What team di the Broncos tie a Super Bowl record for in sacks?'},\n", - " {u'answers': [{u'answer_start': 814,\n", - " u'text': u'two'},\n", - " {u'answer_start': 1199,\n", - " u'text': u'two'},\n", - " {u'answer_start': 1199,\n", - " u'text': u'two'}],\n", - " u'id': u'56d9cd9adc89441400fdb850',\n", - " u'question': u'How many teams has Manning won the Super Bowl with?'}]}],\n", - " u'title': u'Super_Bowl_50'}],\n", - " u'version': u'1.1'}\n" - ] - } - ], - "source": [ - "data = json.load(open('data/tiny-dev.json'))\n", - "\n", - "pprint(data)" - ] - }, - { - "cell_type": "code", - "execution_count": 99, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{u'56be4db0acb8001400a502ec': [u'denver broncos', 0.2304],\n", - " u'56be4db0acb8001400a502ed': [u'denver broncos', 0.2057],\n", - " u'56be4db0acb8001400a502ee': [u\"levi's stadium\", 0.1964],\n", - " u'56be4db0acb8001400a502ef': [u'national football league', 0.0938],\n", - " u'56be4db0acb8001400a502f0': [u'arabic numerals 50', 0.1752],\n", - " u'56be4e1facb8001400a502f6': [u'quarterback cam newton', 0.6254],\n", - " u'56be4e1facb8001400a502f9': [u'eight', 0.8041],\n", - " u'56be4e1facb8001400a502fa': [u'1995', 0.9941],\n", - " u'56be4eafacb8001400a50302': [u'denver linebacker von miller', 0.8649],\n", - " u'56be4eafacb8001400a50303': [u'two', 0.2974],\n", - " u'56be4eafacb8001400a50304': [u'the broncos', 0.6546],\n", - " u'56be5333acb8001400a5030a': [u'cbs', 0.9974],\n", - " u'56be5333acb8001400a5030b': [u'$5 million', 0.9763],\n", - " u'56be5333acb8001400a5030c': [u'british rock group coldplay with special guest performers beyonc\\xe9 and bruno mars',\n", - " 0.1785],\n", - " u'56be5333acb8001400a5030d': [u'coldplay', 0.4255],\n", - " u'56be5333acb8001400a5030e': [u'super bowl xlviii', 0.2898],\n", - " u'56be53b8acb8001400a50314': [u'roger goodell', 0.9977],\n", - " u'56be53b8acb8001400a50315': [u'50th super bowl', 0.508],\n", - " u'56be53b8acb8001400a50316': [u'2012', 0.9757],\n", - " u'56be5438acb8001400a5031a': [u\"new orleans' mercedes-benz superdome\",\n", - " 0.2854],\n", - " u'56be5438acb8001400a5031b': [u\"new orleans' mercedes-benz superdome, miami's sun life stadium\",\n", - " 0.1829],\n", - " u'56be5438acb8001400a5031c': [u\"new orleans' mercedes-benz superdome\",\n", - " 0.0956],\n", - " u'56be54bdacb8001400a50322': [u'october 16, 2012', 0.9351],\n", - " u'56be54bdacb8001400a50323': [u'10', 0.8801],\n", - " u'56be54bdacb8001400a50324': [u'super bowl xliv', 0.8712],\n", - " u'56be54bdacb8001400a50325': [u'2010', 0.9058],\n", - " u'56be54bdacb8001400a50326': [u'1985', 0.8483],\n", - " u'56be5523acb8001400a5032c': [u'may 21, 2013', 0.8779],\n", - " u'56be5523acb8001400a5032d': [u'boston', 0.1976],\n", - " u'56be5523acb8001400a5032e': [u'may 21, 2013', 0.5817],\n", - " u'56be5523acb8001400a5032f': [u'$1.2 billion', 0.7128],\n", - " u'56be5523acb8001400a50330': [u'san diego', 0.2973],\n", - " u'56be572b3aeaaa14008c9052': [u'john fox', 0.99],\n", - " u'56be59683aeaaa14008c9058': [u'deangelo williams', 0.4624],\n", - " u'56be59683aeaaa14008c9059': [u'kelvin benjamin', 0.1915],\n", - " u'56be59683aeaaa14008c905a': [u'15', 0.7218],\n", - " u'56be8e613aeaaa14008c90d1': [u'american football game', 0.3646],\n", - " u'56be8e613aeaaa14008c90d2': [u'february 7, 2016', 0.9091],\n", - " u'56be8e613aeaaa14008c90d3': [u'american football conference', 0.4697],\n", - " u'56bea9923aeaaa14008c91b9': [u'american football game', 0.3646],\n", - " u'56bea9923aeaaa14008c91ba': [u'american football conference', 0.2829],\n", - " u'56bea9923aeaaa14008c91bb': [u'february 7, 2016', 0.6217],\n", - " u'56beaa4a3aeaaa14008c91c2': [u'arizona cardinals 49\\u201315', 0.4563],\n", - " u'56beaa4a3aeaaa14008c91c3': [u'20\\u201318', 0.1812],\n", - " u'56beab833aeaaa14008c91d2': [u'denver linebacker von miller', 0.767],\n", - " u'56beab833aeaaa14008c91d3': [u'five', 0.5837],\n", - " u'56beab833aeaaa14008c91d4': [u'newton', 0.8548],\n", - " u'56beace93aeaaa14008c91df': [u'denver broncos', 0.3552],\n", - " u'56beace93aeaaa14008c91e0': [u\"levi's stadium\", 0.2553],\n", - " u'56beace93aeaaa14008c91e1': [u'california', 0.2178],\n", - " u'56beace93aeaaa14008c91e2': [u'super bowl l', 0.7132],\n", - " u'56beace93aeaaa14008c91e3': [u'2015', 0.4126],\n", - " u'56bead5a3aeaaa14008c91e9': [u'20\\u201318', 0.255],\n", - " u'56bead5a3aeaaa14008c91ea': [u'20\\u201318', 0.5571],\n", - " u'56bead5a3aeaaa14008c91eb': [u'20\\u201318', 0.3996],\n", - " u'56bead5a3aeaaa14008c91ec': [u'four', 0.6316],\n", - " u'56bead5a3aeaaa14008c91ed': [u'quarterback cam newton', 0.7698],\n", - " u'56beae423aeaaa14008c91f4': [u'seven', 0.7592],\n", - " u'56beae423aeaaa14008c91f5': [u'denver linebacker von miller', 0.8325],\n", - " u'56beae423aeaaa14008c91f6': [u'three', 0.4869],\n", - " u'56beae423aeaaa14008c91f7': [u'two', 0.1944],\n", - " u'56beaf5e3aeaaa14008c91fd': [u'cbs', 0.8711],\n", - " u'56beaf5e3aeaaa14008c91fe': [u'$5 million', 0.981],\n", - " u'56beaf5e3aeaaa14008c91ff': [u'bruno mars', 0.4459],\n", - " u'56beaf5e3aeaaa14008c9200': [u'bruno mars', 0.4513],\n", - " u'56beaf5e3aeaaa14008c9201': [u'bruno mars', 0.2329],\n", - " u'56beafca3aeaaa14008c9207': [u'roger goodell', 0.9962],\n", - " u'56beafca3aeaaa14008c9208': [u'early 2012', 0.5371],\n", - " u'56beb03c3aeaaa14008c920b': [u\"new orleans' mercedes-benz superdome, miami's sun life stadium\",\n", - " 0.0993],\n", - " u'56beb03c3aeaaa14008c920d': [u\"levi's stadium\", 0.8002],\n", - " u'56beb0f43aeaaa14008c921b': [u'sun life stadium', 0.977],\n", - " u'56beb0f43aeaaa14008c921c': [u'october 16, 2012', 0.9346],\n", - " u'56beb0f43aeaaa14008c921d': [u'stanford stadium', 0.8133],\n", - " u'56beb0f43aeaaa14008c921e': [u'may 3, 2013', 0.9775],\n", - " u'56beb0f43aeaaa14008c921f': [u'2010', 0.6897],\n", - " u'56beb2153aeaaa14008c9225': [u'boston', 0.5757],\n", - " u'56beb2153aeaaa14008c9226': [u'may 21, 2013', 0.9354],\n", - " u'56beb2153aeaaa14008c9227': [u'$1.2 billion', 0.7128],\n", - " u'56beb2153aeaaa14008c9228': [u'super bowl xxxvii', 0.3464],\n", - " u'56beb2153aeaaa14008c9229': [u'san diego', 0.7718],\n", - " u'56beb2a03aeaaa14008c922f': [u'ten', 0.6692],\n", - " u'56beb2a03aeaaa14008c9230': [u'six', 0.2897],\n", - " u'56beb2a03aeaaa14008c9231': [u'john fox', 0.0836],\n", - " u'56beb2a03aeaaa14008c9232': [u'super bowl xlviii', 0.1967],\n", - " u'56beb2a03aeaaa14008c9233': [u'john fox', 0.9667],\n", - " u'56beb3083aeaaa14008c923d': [u'1978', 0.9951],\n", - " u'56beb3083aeaaa14008c923e': [u'carolina panthers', 0.0907],\n", - " u'56beb3083aeaaa14008c923f': [u'ten', 0.8495],\n", - " u'56beb3083aeaaa14008c9240': [u'eight', 0.9758],\n", - " u'56beb3083aeaaa14008c9241': [u'kelvin benjamin', 0.4348],\n", - " u'56beb3a03aeaaa14008c9247': [u'six', 0.7247],\n", - " u'56beb3a03aeaaa14008c9248': [u'636, while recording a career-high and league-leading 45',\n", - " 0.0892],\n", - " u'56beb3a03aeaaa14008c9249': [u'3,837', 0.1089],\n", - " u'56beb3a03aeaaa14008c924a': [u'27', 0.745],\n", - " u'56beb3a03aeaaa14008c924b': [u'greg olsen', 0.7854],\n", - " u'56beb4343aeaaa14008c925b': [u'308', 0.5732],\n", - " u'56beb4343aeaaa14008c925c': [u'6\\xbd', 0.1154],\n", - " u'56beb4343aeaaa14008c925d': [u'5', 0.0282],\n", - " u'56beb4343aeaaa14008c925e': [u'88', 0.1504],\n", - " u'56beb4343aeaaa14008c925f': [u'mario addison', 0.5089],\n", - " u'56beb4e43aeaaa14008c9265': [u'john fox', 0.3098],\n", - " u'56beb4e43aeaaa14008c9266': [u'brock osweiler', 0.6684],\n", - " u'56beb4e43aeaaa14008c9267': [u'brock osweiler', 0.578],\n", - " u'56beb4e43aeaaa14008c9268': [u'quarterback brock osweiler', 0.5196],\n", - " u'56beb4e43aeaaa14008c9269': [u'john fox', 0.7562],\n", - " u'56beb57b3aeaaa14008c9279': [u'67.9', 0.1606],\n", - " u'56beb57b3aeaaa14008c927a': [u'2,249 yards and nine', 0.0638],\n", - " u'56beb57b3aeaaa14008c927b': [u'demaryius thomas', 0.4716],\n", - " u'56beb57b3aeaaa14008c927c': [u'demaryius thomas', 0.1775],\n", - " u'56beb57b3aeaaa14008c927d': [u'17', 0.1065],\n", - " u'56beb6533aeaaa14008c928d': [u'4,530', 0.1632],\n", - " u'56beb6533aeaaa14008c928e': [u'5\\xbd', 0.4433],\n", - " u'56beb6533aeaaa14008c9290': [u'linebacker brandon marshall', 0.5759],\n", - " u'56beb6533aeaaa14008c9291': [u'three', 0.2758],\n", - " u'56beb6f23aeaaa14008c92a1': [u'arizona cardinals', 0.2421],\n", - " u'56beb6f23aeaaa14008c92a2': [u'arizona cardinals', 0.7072],\n", - " u'56beb6f23aeaaa14008c92a3': [u'487', 0.7313],\n", - " u'56beb6f23aeaaa14008c92a4': [u'seven', 0.1985],\n", - " u'56beb6f23aeaaa14008c92a5': [u'31\\u20130 halftime', 0.2071],\n", - " u'56beb7953aeaaa14008c92ab': [u'pittsburgh', 0.2199],\n", - " u'56beb7953aeaaa14008c92ac': [u'11', 0.7719],\n", - " u'56beb7953aeaaa14008c92ad': [u'new england patriots', 0.1764],\n", - " u'56beb7953aeaaa14008c92ae': [u'11 points', 0.2187],\n", - " u'56beb7953aeaaa14008c92af': [u'17 seconds', 0.8691],\n", - " u'56beb7fd3aeaaa14008c92b5': [u'thomas davis', 0.8903],\n", - " u'56beb7fd3aeaaa14008c92b6': [u'a broken arm', 0.2065],\n", - " u'56beb7fd3aeaaa14008c92b7': [u'three', 0.9859],\n", - " u'56beb7fd3aeaaa14008c92b8': [u'three', 0.2286],\n", - " u'56beb86b3aeaaa14008c92bd': [u'39', 0.6623],\n", - " u'56beb86b3aeaaa14008c92be': [u'john elway', 0.9716],\n", - " u'56beb86b3aeaaa14008c92bf': [u'38', 0.3817],\n", - " u'56beb86b3aeaaa14008c92c0': [u'executive vice president of football operations and general manager',\n", - " 0.1768],\n", - " u'56beb86b3aeaaa14008c92c1': [u\"age 38 and is currently denver's executive vice president of football operations and general manager\",\n", - " 0.1069],\n", - " u'56beb90c3aeaaa14008c92c7': [u'1998', 0.7265],\n", - " u'56beb90c3aeaaa14008c92c8': [u'2011', 0.5492],\n", - " u'56beb90c3aeaaa14008c92c9': [u'26', 0.6226],\n", - " u'56beb90c3aeaaa14008c92ca': [u'26', 0.3003],\n", - " u'56beb90c3aeaaa14008c92cb': [u'manning', 0.4863],\n", - " u'56beb97c3aeaaa14008c92db': [u'super bowl xx', 0.5341],\n", - " u'56beb97c3aeaaa14008c92dc': [u'chicago bears', 0.3659],\n", - " u'56beb97c3aeaaa14008c92dd': [u'linebacker', 0.6956],\n", - " u'56beb97c3aeaaa14008c92de': [u'elway', 0.8384],\n", - " u'56beb97c3aeaaa14008c92df': [u'chicago bears', 0.0656],\n", - " u'56beba103aeaaa14008c92e5': [u'justin tucker', 0.5819],\n", - " u'56beba103aeaaa14008c92e6': [u'419 turf', 0.1624],\n", - " u'56beba103aeaaa14008c92e7': [u'ed mangan', 0.9904],\n", - " u'56beba103aeaaa14008c92e8': [u'baltimore ravens', 0.4413],\n", - " u'56beba103aeaaa14008c92e9': [u'kicker', 0.2562],\n", - " u'56bebad93aeaaa14008c92f9': [u'pittsburgh steelers', 0.6018],\n", - " u'56bebad93aeaaa14008c92fa': [u'denver qb', 0.1579],\n", - " u'56bebad93aeaaa14008c92fb': [u'wearing orange jerseys', 0.1124],\n", - " u'56bebad93aeaaa14008c92fc': [u'white', 0.38],\n", - " u'56bebad93aeaaa14008c92fd': [u'super bowl xxxiii', 0.3475],\n", - " u'56bebb293aeaaa14008c9303': [u'stanford university', 0.4229],\n", - " u'56bebb293aeaaa14008c9304': [u'stanford university', 0.4501],\n", - " u'56bebb293aeaaa14008c9305': [u'san jose marriott', 0.1752],\n", - " u'56bebb293aeaaa14008c9306': [u'santa clara marriott', 0.247],\n", - " u'56bebbbf3aeaaa14008c9315': [u'june 4, 2014', 0.9512],\n", - " u'56bebbbf3aeaaa14008c9316': [u'super bowl', 0.1651],\n", - " u'56bebbbf3aeaaa14008c9317': [u'jaime weston', 0.994],\n", - " u'56bebbbf3aeaaa14008c9318': [u'super bowl xlv', 0.8322],\n", - " u'56bebbbf3aeaaa14008c9319': [u'vince lombardi', 0.4283],\n", - " u'56bebc383aeaaa14008c931f': [u'gold', 0.2169],\n", - " u'56bebc383aeaaa14008c9320': [u'week 7', 0.6901],\n", - " u'56bebc383aeaaa14008c9321': [u'50-yard line', 0.5107],\n", - " u'56bebcbe3aeaaa14008c9325': [u'moscone center', 0.6493],\n", - " u'56bebcbe3aeaaa14008c9326': [u'san francisco', 0.9817],\n", - " u'56bebcbe3aeaaa14008c9327': [u'ed lee', 0.8045],\n", - " u'56bebcbe3aeaaa14008c9328': [u'jane kim', 0.9907],\n", - " u'56bebcbe3aeaaa14008c9329': [u'january 30', 0.9895],\n", - " u'56bebd713aeaaa14008c932f': [u'santa clara university', 0.7881],\n", - " u'56bebd713aeaaa14008c9330': [u'$2 million', 0.7462],\n", - " u'56bebd713aeaaa14008c9331': [u'week-long', 0.8378],\n", - " u'56bebde53aeaaa14008c9335': [u\"game's media day, which was typically held on the tuesday afternoon\",\n", - " 0.0615],\n", - " u'56bebde53aeaaa14008c9336': [u'night', 0.1076],\n", - " u'56bebde53aeaaa14008c9337': [u'sap center', 0.1018],\n", - " u'56bebde53aeaaa14008c9338': [u'san jose', 0.9729],\n", - " u'56bebde53aeaaa14008c9339': [u'golden gate bridge', 0.5243],\n", - " u'56bebe873aeaaa14008c933f': [u'super bowl program', 0.4459],\n", - " u'56bebe873aeaaa14008c9340': [u'over $40 million', 0.6036],\n", - " u'56bebe873aeaaa14008c9341': [u'apple, google, yahoo!, intel, gap, chevron, and dignity health',\n", - " 0.0372],\n", - " u'56bebe873aeaaa14008c9342': [u'transgender-owned', 0.0557],\n", - " u'56bebe873aeaaa14008c9343': [u'apple, google, yahoo', 0.0191],\n", - " u'56bebec43aeaaa14008c9349': [u'25', 0.4858],\n", - " u'56bebec43aeaaa14008c934a': [u'the 50 fund', 0.4265],\n", - " u'56bec0353aeaaa14008c934d': [u'vince lombardi trophy', 0.4196],\n", - " u'56bec0353aeaaa14008c934e': [u'66 lb', 0.0674],\n", - " u'56bec0353aeaaa14008c934f': [u'66 lb', 0.041],\n", - " u'56bec0353aeaaa14008c9350': [u'tiffany & co', 0.9178],\n", - " u'56bec0353aeaaa14008c9351': [u'tiffany & co', 0.9594],\n", - " u'56bec0dd3aeaaa14008c9357': [u'cbs', 0.9575],\n", - " u'56bec0dd3aeaaa14008c9358': [u'phil simms', 0.4979],\n", - " u'56bec0dd3aeaaa14008c9359': [u'tracy wolfson', 0.99],\n", - " u'56bec0dd3aeaaa14008c935a': [u'36', 0.8632],\n", - " u'56bec0dd3aeaaa14008c935b': [u'5k', 0.6452],\n", - " u'56bec1823aeaaa14008c9361': [u'espn deportes', 0.0446],\n", - " u'56bec1823aeaaa14008c9362': [u'john sutcliffe', 0.9409],\n", - " u'56bec1823aeaaa14008c9363': [u'john sutcliffe', 0.4449],\n", - " u'56bec1823aeaaa14008c9364': [u'december 28, 2015', 0.9881],\n", - " u'56bec1823aeaaa14008c9365': [u'spanish', 0.6837],\n", - " u'56bec2013aeaaa14008c9371': [u'nfl mobile', 0.3337],\n", - " u'56bec2013aeaaa14008c9372': [u'watchespn', 0.7689],\n", - " u'56bec2013aeaaa14008c9373': [u'cbssports.com', 0.9131],\n", - " u'56bec2013aeaaa14008c9374': [u'tablets', 0.6969],\n", - " u'56bec2013aeaaa14008c9375': [u'10', 0.2306],\n", - " u'56bec2573aeaaa14008c937b': [u'night talk', 0.155],\n", - " u'56bec2573aeaaa14008c937c': [u'james corden', 0.385],\n", - " u'56bec3153aeaaa14008c9389': [u'$5,000,000', 0.7962],\n", - " u'56bec3153aeaaa14008c938a': [u'inbev', 0.581],\n", - " u'56bec3153aeaaa14008c938b': [u'doritos', 0.9453],\n", - " u'56bec3153aeaaa14008c938c': [u'pok\\xe9mon video game', 0.2661],\n", - " u'56bec38b3aeaaa14008c9397': [u'small business big game', 0.623],\n", - " u'56bec38b3aeaaa14008c9398': [u'death wish coffee', 0.2597],\n", - " u'56bec38b3aeaaa14008c9399': [u'nine', 0.6918],\n", - " u'56bec38b3aeaaa14008c939a': [u'nine', 0.9189],\n", - " u'56bec4343aeaaa14008c93a9': [u'paramount pictures', 0.0621],\n", - " u'56bec4343aeaaa14008c93aa': [u'walt disney studios', 0.1675],\n", - " u'56bec4343aeaaa14008c93ab': [u'shadows', 0.0514],\n", - " u'56bec4343aeaaa14008c93ac': [u'apocalypse', 0.0695],\n", - " u'56bec4a33aeaaa14008c93b1': [u'westwood', 0.7223],\n", - " u'56bec4a33aeaaa14008c93b2': [u'kevin harlan', 0.7423],\n", - " u'56bec4a33aeaaa14008c93b3': [u'jim gray', 0.867],\n", - " u'56bec4a33aeaaa14008c93b4': [u'boomer esiason', 0.2471],\n", - " u'56bec4a33aeaaa14008c93b5': [u'james lofton', 0.9462],\n", - " u'56bec5233aeaaa14008c93cf': [u'103.5 fm', 0.2866],\n", - " u'56bec5233aeaaa14008c93d0': [u'dave logan', 0.4446],\n", - " u'56bec5233aeaaa14008c93d1': [u'wbt-fm', 0.2037],\n", - " u'56bec5233aeaaa14008c93d2': [u'denver', 0.7215],\n", - " u'56bec5233aeaaa14008c93d3': [u'mick mixon', 0.6852],\n", - " u'56bec5ff3aeaaa14008c93e3': [u'5', 0.2799],\n", - " u'56bec5ff3aeaaa14008c93e4': [u'5', 0.1524],\n", - " u'56bec5ff3aeaaa14008c93e5': [u'the bbc', 0.1267],\n", - " u'56bec6573aeaaa14008c93e9': [u'bart starr', 0.7509],\n", - " u'56bec6573aeaaa14008c93ea': [u'chuck howley', 0.9439],\n", - " u'56bec6573aeaaa14008c93eb': [u'bart starr', 0.2596],\n", - " u'56bec6573aeaaa14008c93ec': [u'2001', 0.9967],\n", - " u'56bec6573aeaaa14008c93ed': [u'peyton manning', 0.8981],\n", - " u'56bec6ac3aeaaa14008c93fd': [u'six-time grammy winner', 0.1905],\n", - " u'56bec6ac3aeaaa14008c93fe': [u'national anthem', 0.6745],\n", - " u'56bec6ac3aeaaa14008c93ff': [u'academy award', 0.4203],\n", - " u'56bec6ac3aeaaa14008c9400': [u'american sign language', 0.5065],\n", - " u'56bec6ac3aeaaa14008c9401': [u'american sign language ( asl', 0.1851],\n", - " u'56bec7353aeaaa14008c940d': [u'january 7, 2016', 0.5164],\n", - " u'56bec7353aeaaa14008c940e': [u'british', 0.9825],\n", - " u'56bec7353aeaaa14008c940f': [u'super bowl xlvii', 0.4341],\n", - " u'56bec7353aeaaa14008c9410': [u'hymn for the weekend', 0.5369],\n", - " u'56bec7353aeaaa14008c9411': [u'super bowl xlviii', 0.2914],\n", - " u'56bec7a63aeaaa14008c9417': [u'denver', 0.5081],\n", - " u'56bec7a63aeaaa14008c9418': [u'owen daniels', 0.3331],\n", - " u'56bec7a63aeaaa14008c9419': [u'c. j. anderson', 0.1844],\n", - " u'56bec7a63aeaaa14008c941a': [u'brandon mcmanus', 0.3437],\n", - " u'56bec7a63aeaaa14008c941b': [u'c. j. anderson', 0.6325],\n", - " u'56bec8243aeaaa14008c942b': [u'mike carey', 0.98],\n", - " u'56bec8243aeaaa14008c942c': [u'panthers quarterback cam newton', 0.3826],\n", - " u'56bec8243aeaaa14008c942d': [u'linebacker von miller', 0.788],\n", - " u'56bec8243aeaaa14008c942e': [u'malik jackson', 0.9433],\n", - " u'56bec8243aeaaa14008c942f': [u'super bowl xxviii', 0.85],\n", - " u'56bec8a13aeaaa14008c9435': [u'jonathan stewart', 0.9791],\n", - " u'56bec8a13aeaaa14008c9436': [u'jonathan stewart', 0.8889],\n", - " u'56bec8a13aeaaa14008c9437': [u'25', 0.357],\n", - " u'56bec8a13aeaaa14008c9438': [u'25', 0.4525],\n", - " u'56bec8a13aeaaa14008c9439': [u'25', 0.1632],\n", - " u'56bec9133aeaaa14008c9445': [u'safety darian stewart', 0.7674],\n", - " u'56bec9133aeaaa14008c9446': [u'linebacker', 0.8635],\n", - " u'56bec9133aeaaa14008c9447': [u'kony ealy', 0.9767],\n", - " u'56bec9133aeaaa14008c9448': [u'demarcus ware', 0.8259],\n", - " u'56bec9133aeaaa14008c9449': [u'demarcus ware', 0.9658],\n", - " u'56bec98e3aeaaa14008c9455': [u'ted ginn jr.', 0.1134],\n", - " u'56bec98e3aeaaa14008c9456': [u'graham gano', 0.8671],\n", - " u'56bec98e3aeaaa14008c9457': [u'22', 0.5744],\n", - " u'56bec98e3aeaaa14008c9458': [u'graham gano', 0.7168],\n", - " u'56bec98e3aeaaa14008c9459': [u'trevathan', 0.1172],\n", - " u'56bec9e83aeaaa14008c945f': [u'devin funchess', 0.2294],\n", - " u'56bec9e83aeaaa14008c9460': [u'one', 0.2986],\n", - " u'56bec9e83aeaaa14008c9461': [u'devin funchess and a 12-yard run by stewart',\n", - " 0.2215],\n", - " u'56bec9e83aeaaa14008c9462': [u'devin funchess', 0.2017],\n", - " u'56beca913aeaaa14008c946d': [u'24-yard line', 0.4041],\n", - " u'56beca913aeaaa14008c946e': [u'carolina', 0.1062],\n", - " u'56beca913aeaaa14008c946f': [u'cornerback josh norman', 0.207],\n", - " u'56beca913aeaaa14008c9470': [u'miller', 0.4712],\n", - " u'56beca913aeaaa14008c9471': [u'bennie fowler', 0.9505],\n", - " u'56becaf93aeaaa14008c9481': [u'five', 0.3295],\n", - " u'56becaf93aeaaa14008c9482': [u'three', 0.0292],\n", - " u'56becaf93aeaaa14008c9483': [u'four', 0.475],\n", - " u'56becaf93aeaaa14008c9484': [u'thomas davis', 0.8681],\n", - " u'56becaf93aeaaa14008c9485': [u'six', 0.0545],\n", - " u'56becb823aeaaa14008c948b': [u'194', 0.1387],\n", - " u'56becb823aeaaa14008c948c': [u'11', 0.3708],\n", - " u'56becb823aeaaa14008c948d': [u'baltimore ravens', 0.8578],\n", - " u'56becb823aeaaa14008c948e': [u'jordan norwood', 0.0646],\n", - " u'56becb823aeaaa14008c948f': [u'gary kubiak', 0.9279],\n", - " u'56bf10f43aeaaa14008c94fd': [u'2015', 0.7326],\n", - " u'56bf10f43aeaaa14008c94fe': [u'california', 0.2178],\n", - " u'56bf10f43aeaaa14008c94ff': [u\"levi's stadium\", 0.3289],\n", - " u'56bf10f43aeaaa14008c9500': [u'2015 season', 0.1286],\n", - " u'56bf10f43aeaaa14008c9501': [u'february 7, 2016', 0.2174],\n", - " u'56bf159b3aeaaa14008c9507': [u'12\\u20134', 0.5877],\n", - " u'56bf159b3aeaaa14008c9508': [u'quarterback cam newton', 0.5049],\n", - " u'56bf159b3aeaaa14008c9509': [u'12\\u20134', 0.3951],\n", - " u'56bf159b3aeaaa14008c950a': [u'four', 0.4796],\n", - " u'56bf159b3aeaaa14008c950b': [u'new england patriots', 0.3937],\n", - " u'56bf17653aeaaa14008c9511': [u'denver linebacker von miller', 0.349],\n", - " u'56bf17653aeaaa14008c9513': [u'super bowl mvp', 0.3179],\n", - " u'56bf17653aeaaa14008c9514': [u'five', 0.3813],\n", - " u'56bf17653aeaaa14008c9515': [u'two', 0.9063],\n", - " u'56bf1ae93aeaaa14008c951b': [u'coldplay', 0.4884],\n", - " u'56bf1ae93aeaaa14008c951c': [u'$5 million', 0.9755],\n", - " u'56bf1ae93aeaaa14008c951e': [u'bruno mars', 0.4393],\n", - " u'56bf1ae93aeaaa14008c951f': [u'third-most', 0.6686],\n", - " u'56bf21b43aeaaa14008c9525': [u'two', 0.6452],\n", - " u'56bf21b43aeaaa14008c9526': [u'super bowl xliv', 0.4039],\n", - " u'56bf21b43aeaaa14008c9528': [u'two', 0.9555],\n", - " u'56bf21b43aeaaa14008c9529': [u'funding plan', 0.051],\n", - " u'56bf23363aeaaa14008c952f': [u'2013', 0.2218],\n", - " u'56bf23363aeaaa14008c9530': [u'may 21, 2013', 0.8676],\n", - " u'56bf23363aeaaa14008c9531': [u'$1.2 billion', 0.6556],\n", - " u'56bf23363aeaaa14008c9532': [u'1985', 0.9514],\n", - " u'56bf23363aeaaa14008c9533': [u'super bowl xxxvii', 0.4695],\n", - " u'56bf28c73aeaaa14008c9539': [u'eight', 0.2459],\n", - " u'56bf28c73aeaaa14008c953a': [u'ten', 0.5535],\n", - " u'56bf28c73aeaaa14008c953c': [u'super bowl xxxviii', 0.2634],\n", - " u'56bf28c73aeaaa14008c953d': [u'six', 0.6812],\n", - " u'56bf2afe3aeaaa14008c9543': [u'1978', 0.9904],\n", - " u'56bf2afe3aeaaa14008c9544': [u'2009', 0.2975],\n", - " u'56bf2afe3aeaaa14008c9545': [u'1978', 0.3242],\n", - " u'56bf2afe3aeaaa14008c9547': [u'acl', 0.376],\n", - " u'56bf301c3aeaaa14008c954d': [u'four', 0.8717],\n", - " u'56bf301c3aeaaa14008c954e': [u'gary kubiak', 0.6596],\n", - " u'56bf301c3aeaaa14008c954f': [u'indianapolis colts', 0.1785],\n", - " u'56bf301c3aeaaa14008c9550': [u'39', 0.9811],\n", - " u'56bf301c3aeaaa14008c9551': [u'plantar fasciitis', 0.5631],\n", - " u'56bf34293aeaaa14008c9559': [u'arizona cardinals', 0.4246],\n", - " u'56bf34293aeaaa14008c955a': [u'487 yards', 0.3916],\n", - " u'56bf34293aeaaa14008c955b': [u'487', 0.7309],\n", - " u'56bf36b93aeaaa14008c9561': [u'pittsburgh', 0.2816],\n", - " u'56bf36b93aeaaa14008c9562': [u'11 points', 0.4084],\n", - " u'56bf36b93aeaaa14008c9563': [u'new england patriots', 0.1764],\n", - " u'56bf36b93aeaaa14008c9564': [u'17', 0.2605],\n", - " u'56bf36b93aeaaa14008c9565': [u\"manning's problems with interceptions\",\n", - " 0.2425],\n", - " u'56bf38383aeaaa14008c956b': [u'67.9', 0.222],\n", - " u'56bf38383aeaaa14008c956c': [u'2,249', 0.4178],\n", - " u'56bf38383aeaaa14008c956d': [u'2,249 yards and nine touchdowns', 0.0862],\n", - " u'56bf38383aeaaa14008c956e': [u'manning', 0.667],\n", - " u'56bf38383aeaaa14008c956f': [u'veteran', 0.5603],\n", - " u'56bf3a223aeaaa14008c9575': [u'acl tears', 0.1022],\n", - " u'56bf3a223aeaaa14008c9576': [u'broken arm', 0.3863],\n", - " u'56bf3a223aeaaa14008c9577': [u'three', 0.2286],\n", - " u'56bf3a223aeaaa14008c9579': [u'super bowl', 0.6986],\n", - " u'56bf3c633aeaaa14008c9580': [u'san francisco bay', 0.2942],\n", - " u'56bf3c633aeaaa14008c9581': [u\"new orleans' mercedes-benz superdome\", 0.34],\n", - " u'56bf3c633aeaaa14008c9582': [u'sun life stadium', 0.1052],\n", - " u'56bf3e803aeaaa14008c9588': [u'pro bowl', 0.1717],\n", - " u'56bf3e803aeaaa14008c9589': [u'pro bowl', 0.0622],\n", - " u'56bf3e803aeaaa14008c958b': [u'malik jackson', 0.2225],\n", - " u'56bf3fd53aeaaa14008c9591': [u'john elway', 0.4023],\n", - " u'56bf3fd53aeaaa14008c9592': [u'john elway', 0.1699],\n", - " u'56bf3fd53aeaaa14008c9593': [u'38', 0.4604],\n", - " u'56bf3fd53aeaaa14008c9594': [u'john elway', 0.9625],\n", - " u'56bf3fd53aeaaa14008c9595': [u'two', 0.732],\n", - " u'56bf41013aeaaa14008c959b': [u'manning', 0.7089],\n", - " u'56bf41013aeaaa14008c959c': [u'newton', 0.4089],\n", - " u'56bf41013aeaaa14008c959d': [u'26', 0.6226],\n", - " u'56bf41013aeaaa14008c959e': [u'26', 0.045],\n", - " u'56bf42f53aeaaa14008c95a3': [u'roger goodell', 0.9973],\n", - " u'56bf467d3aeaaa14008c95a5': [u'justin tucker', 0.5314],\n", - " u'56bf467d3aeaaa14008c95a6': [u'natural grass stadiums', 0.1303],\n", - " u'56bf467d3aeaaa14008c95a7': [u'concerns', 0.0746],\n", - " u'56bf48cc3aeaaa14008c95ab': [u'denver qb', 0.9294],\n", - " u'56bf48cc3aeaaa14008c95ac': [u'denver qb', 0.103],\n", - " u'56bf48cc3aeaaa14008c95ae': [u'atlanta falcons 34\\u201319', 0.0175],\n", - " u'56bf48cc3aeaaa14008c95af': [u'pants', 0.1117],\n", - " u'56bf49993aeaaa14008c95b5': [u'san jose marriott', 0.5681],\n", - " u'56bf49993aeaaa14008c95b6': [u'san jose marriott', 0.6592],\n", - " u'56bf49993aeaaa14008c95b7': [u'stanford university', 0.3449],\n", - " u'56bf49993aeaaa14008c95b8': [u'stanford university', 0.5058],\n", - " u'56bf4e1b3aeaaa14008c95bd': [u'2014', 0.9679],\n", - " u'56bf4e1b3aeaaa14008c95be': [u'super bowl', 0.2921],\n", - " u'56bf4e1b3aeaaa14008c95c0': [u'super bowl li', 0.205],\n", - " u'56bf4e1b3aeaaa14008c95c1': [u'silver', 0.5703],\n", - " u'56bf52483aeaaa14008c95c7': [u'colored gold', 0.7188],\n", - " u'56bf52483aeaaa14008c95c8': [u'gold', 0.9234],\n", - " u'56bf53e73aeaaa14008c95cb': [u'more than 1 million', 0.6358],\n", - " u'56bf53e73aeaaa14008c95cc': [u'ed lee', 0.725],\n", - " u'56bf53e73aeaaa14008c95cd': [u'moscone center', 0.6981],\n", - " u'56bf53e73aeaaa14008c95ce': [u'super bowl city', 0.0889],\n", - " u'56bf555e3aeaaa14008c95d3': [u'$2 million', 0.7616],\n", - " u'56bf555e3aeaaa14008c95d4': [u'pep', 0.3236],\n", - " u'56bf555e3aeaaa14008c95d5': [u'city council', 0.4108],\n", - " u'56bf57043aeaaa14008c95d9': [u'night', 0.29],\n", - " u'56bf57043aeaaa14008c95da': [u'monday evening and re-branded as super bowl opening night',\n", - " 0.087],\n", - " u'56bf57043aeaaa14008c95db': [u'availabilities', 0.4327],\n", - " u'56bf57043aeaaa14008c95dc': [u'availabilities', 0.1795],\n", - " u'56bf57043aeaaa14008c95dd': [u'san jose', 0.3688],\n", - " u'56bf59523aeaaa14008c95e3': [u'super bowl 50', 0.0763],\n", - " u'56bf59523aeaaa14008c95e4': [u'over $40 million', 0.5788],\n", - " u'56bf59523aeaaa14008c95e5': [u'over $40 million', 0.1579],\n", - " u'56bf5abc3aeaaa14008c95e9': [u'25 percent', 0.6342],\n", - " u'56bf5abc3aeaaa14008c95ea': [u'the 50 fund', 0.287],\n", - " u'56bf5b933aeaaa14008c95ed': [u'vince lombardi trophy', 0.2597],\n", - " u'56bf5b933aeaaa14008c95ef': [u'66 lb', 0.1081],\n", - " u'56bf5b933aeaaa14008c95f1': [u'tiffany & co', 0.7228],\n", - " u'56bf5e7b3aeaaa14008c95f7': [u'three', 0.9231],\n", - " u'56bf5e7b3aeaaa14008c95f8': [u'on the sidelines', 0.4048],\n", - " u'56bf5e7b3aeaaa14008c95f9': [u'360-degree', 0.3146],\n", - " u'56bf5e7b3aeaaa14008c95fa': [u'5k resolution', 0.6268],\n", - " u'56bf5e7b3aeaaa14008c95fb': [u'on the sidelines', 0.4335],\n", - " u'56bf6b303aeaaa14008c960b': [u'636, while recording a career-high and league-leading 45',\n", - " 0.0951],\n", - " u'56bf6b303aeaaa14008c960c': [u'99.4', 0.9103],\n", - " u'56bf6b303aeaaa14008c960d': [u'career-high 77 passes for 1,104 yards and seven touchdowns',\n", - " 0.0998],\n", - " u'56bf6b303aeaaa14008c960e': [u'veteran', 0.6147],\n", - " u'56bf6b303aeaaa14008c960f': [u'mike tolbert', 0.3748],\n", - " u'56bf6c3e3aeaaa14008c9615': [u'linebacker', 0.6956],\n", - " u'56bf6c3e3aeaaa14008c9617': [u'elway', 0.6872],\n", - " u'56bf6d343aeaaa14008c961b': [u'espn deportes', 0.4568],\n", - " u'56bf6e7a3aeaaa14008c961f': [u'smartphones', 0.6128],\n", - " u'56bf6e7a3aeaaa14008c9620': [u'smartphones', 0.598],\n", - " u'56bf6e7a3aeaaa14008c9621': [u'verizon', 0.4189],\n", - " u'56bf6e7a3aeaaa14008c9622': [u'cbssports.com', 0.0908],\n", - " u'56bf6f743aeaaa14008c9631': [u'stephen colbert', 0.4678],\n", - " u'56bf6f743aeaaa14008c9632': [u'stephen colbert', 0.5151],\n", - " u'56bf717d3aeaaa14008c9635': [u'$5,000,000', 0.9159],\n", - " u'56bf717d3aeaaa14008c9637': [u'anheuser-busch inbev', 0.463],\n", - " u'56bf717d3aeaaa14008c9638': [u'nintendo and the pok\\xe9mon', 0.1497],\n", - " u'56bf717d3aeaaa14008c9639': [u'nintendo and the pok\\xe9mon', 0.2199],\n", - " u'56bf71e73aeaaa14008c963f': [u'quickbooks', 0.5438],\n", - " u'56bf71e73aeaaa14008c9640': [u'nine', 0.9081],\n", - " u'56bf738b3aeaaa14008c9655': [u'color', 0.3647],\n", - " u'56bf738b3aeaaa14008c9656': [u'kevin harlan', 0.904],\n", - " u'56d20362e7d4791d009025e8': [u'2015', 0.8597],\n", - " u'56d20362e7d4791d009025e9': [u'denver broncos', 0.671],\n", - " u'56d20362e7d4791d009025ea': [u'denver broncos', 0.1472],\n", - " u'56d20362e7d4791d009025eb': [u'denver broncos', 0.3552],\n", - " u'56d2045de7d4791d009025f3': [u'cam newton', 0.5771],\n", - " u'56d2045de7d4791d009025f4': [u'cardinals 49\\u201315', 0.2605],\n", - " u'56d2045de7d4791d009025f5': [u'eight', 0.5145],\n", - " u'56d2045de7d4791d009025f6': [u'20\\u201318', 0.5],\n", - " u'56d204ade7d4791d00902603': [u'denver linebacker von miller', 0.5628],\n", - " u'56d204ade7d4791d00902604': [u'2\\xbd', 0.1377],\n", - " u'56d2051ce7d4791d00902608': [u'cbs', 0.4219],\n", - " u'56d2051ce7d4791d00902609': [u'$5 million', 0.8053],\n", - " u'56d2051ce7d4791d0090260a': [u'coldplay', 0.4759],\n", - " u'56d2051ce7d4791d0090260b': [u'beyonc\\xe9 and bruno mars', 0.7505],\n", - " u'56d2053ae7d4791d00902610': [u'roger goodell', 0.995],\n", - " u'56d20564e7d4791d00902612': [u\"new orleans' mercedes-benz superdome, miami's sun life stadium\",\n", - " 0.2813],\n", - " u'56d20650e7d4791d00902614': [u'lady gaga', 0.9661],\n", - " u'56d20650e7d4791d00902615': [u'marlee matlin', 0.7743],\n", - " u'56d600e31c85041400946eae': [u'2015', 0.3952],\n", - " u'56d600e31c85041400946eb0': [u'denver broncos', 0.1134],\n", - " u'56d600e31c85041400946eb1': [u\"levi's stadium\", 0.1158],\n", - " u'56d6017d1c85041400946ebe': [u'quarterback cam newton', 0.918],\n", - " u'56d6017d1c85041400946ec1': [u'20\\u201318', 0.4084],\n", - " u'56d6017d1c85041400946ec2': [u'arizona cardinals 49\\u201315', 0.1107],\n", - " u'56d601e41c85041400946ece': [u'seven', 0.7701],\n", - " u'56d601e41c85041400946ecf': [u'seven', 0.3691],\n", - " u'56d601e41c85041400946ed0': [u'fumble', 0.6811],\n", - " u'56d601e41c85041400946ed1': [u'the broncos', 0.394],\n", - " u'56d601e41c85041400946ed2': [u'denver linebacker', 0.2714],\n", - " u'56d602631c85041400946ed8': [u'bruno mars', 0.1328],\n", - " u'56d602631c85041400946eda': [u'coldplay with special guest performers beyonc\\xe9 and bruno mars',\n", - " 0.2714],\n", - " u'56d602631c85041400946edb': [u'beyonc\\xe9 and bruno mars', 0.5494],\n", - " u'56d602631c85041400946edc': [u'super bowl xlvii and super bowl xlviii',\n", - " 0.537],\n", - " u'56d6edd00d65d21400198250': [u'roger goodell', 0.9978],\n", - " u'56d6edd00d65d21400198251': [u'an important game for us as a league \"',\n", - " 0.1603],\n", - " u'56d6ee6e0d65d21400198254': [u'three', 0.6843],\n", - " u'56d6ee6e0d65d21400198255': [u\"miami's sun life stadium\", 0.5696],\n", - " u'56d6ee6e0d65d21400198256': [u'sun life stadium', 0.2966],\n", - " u'56d6ee6e0d65d21400198257': [u\"new orleans' mercedes-benz superdome\",\n", - " 0.2257],\n", - " u'56d6ee6e0d65d21400198258': [u\"levi's stadium\", 0.6509],\n", - " u'56d6ef6a0d65d21400198260': [u'1985', 0.8395],\n", - " u'56d6ef6a0d65d21400198262': [u'new orleans', 0.1044],\n", - " u'56d6f0770d65d21400198268': [u'1985', 0.7703],\n", - " u'56d6f0770d65d21400198269': [u'may 21, 2013', 0.8289],\n", - " u'56d6f0770d65d2140019826a': [u'2003', 0.2953],\n", - " u'56d6f0770d65d2140019826c': [u'boston', 0.3307],\n", - " u'56d6f1190d65d21400198272': [u'denver broncos', 0.0601],\n", - " u'56d6f1190d65d21400198273': [u'super bowl xlviii', 0.103],\n", - " u'56d6f1190d65d21400198274': [u'three years', 0.2342],\n", - " u'56d6f1190d65d21400198275': [u'in their last super bowl appearance', 0.0752],\n", - " u'56d6f1190d65d21400198276': [u'six', 0.348],\n", - " u'56d6f2000d65d2140019827c': [u'kelvin benjamin', 0.7936],\n", - " u'56d6f2000d65d2140019827d': [u'deangelo williams', 0.9361],\n", - " u'56d6f2000d65d2140019827e': [u'1978', 0.8019],\n", - " u'56d6f2000d65d2140019827f': [u'ten', 0.7652],\n", - " u'56d6f2960d65d21400198286': [u'six', 0.1882],\n", - " u'56d6f2960d65d21400198287': [u'cam newton', 0.516],\n", - " u'56d6f2960d65d21400198288': [u'3,837', 0.1585],\n", - " u'56d6f2960d65d21400198289': [u'10', 0.0535],\n", - " u'56d6f2960d65d2140019828a': [u'six', 0.5019],\n", - " u'56d6f3500d65d21400198290': [u'four', 0.2436],\n", - " u'56d6f3500d65d21400198291': [u'kuechly', 0.1415],\n", - " u'56d6f3500d65d21400198292': [u'two', 0.4391],\n", - " u'56d6f3500d65d21400198293': [u'four', 0.6902],\n", - " u'56d6f3500d65d21400198294': [u'mario addison', 0.3066],\n", - " u'56d6f4030d65d2140019829a': [u'gary kubiak', 0.9672],\n", - " u'56d6f4030d65d2140019829b': [u'brock osweiler', 0.6664],\n", - " u'56d6f4030d65d2140019829c': [u'plantar fasciitis', 0.6188],\n", - " u'56d6f4030d65d2140019829d': [u'39', 0.9608],\n", - " u'56d6f4030d65d2140019829e': [u'39', 0.322],\n", - " u'56d6fe0b0d65d214001982a4': [u'86.4', 0.1899],\n", - " u'56d6fe0b0d65d214001982a5': [u'17', 0.0667],\n", - " u'56d6fe0b0d65d214001982a6': [u'demaryius thomas', 0.7812],\n", - " u'56d6fe0b0d65d214001982a7': [u'17', 0.0806],\n", - " u'56d6fea90d65d214001982ae': [u'296', 0.1189],\n", - " u'56d6fea90d65d214001982af': [u'linebacker demarcus ware', 0.1225],\n", - " u'56d6fea90d65d214001982b0': [u'linebacker brandon marshall', 0.7657],\n", - " u'56d6fea90d65d214001982b2': [u'three', 0.2785],\n", - " u'56d701280d65d214001982b8': [u'seahawks', 0.4527],\n", - " u'56d701280d65d214001982ba': [u'arizona cardinals', 0.3695],\n", - " u'56d701280d65d214001982bc': [u'seven', 0.847],\n", - " u'56d7018a0d65d214001982c2': [u'pittsburgh steelers', 0.1709],\n", - " u'56d7018a0d65d214001982c3': [u'11', 0.7641],\n", - " u'56d7018a0d65d214001982c5': [u'pittsburgh steelers', 0.3023],\n", - " u'56d703d10d65d214001982d6': [u'three', 0.9393],\n", - " u'56d703d10d65d214001982d7': [u'a broken arm', 0.1108],\n", - " u'56d703d10d65d214001982d8': [u'three', 0.9485],\n", - " u'56d704430d65d214001982de': [u'two', 0.9735],\n", - " u'56d704430d65d214001982e0': [u'peyton manning', 0.8849],\n", - " u'56d704430d65d214001982e1': [u'peyton manning', 0.7149],\n", - " u'56d704430d65d214001982e2': [u'super bowl xxxiii', 0.8188],\n", - " u'56d7096b0d65d214001982fc': [u'1998 nfl draft', 0.1041],\n", - " u'56d7096b0d65d214001982fd': [u'2011', 0.0622],\n", - " u'56d7096b0d65d214001982fe': [u'newton for carolina and von miller', 0.3649],\n", - " u'56d709ef0d65d21400198306': [u'kubiak', 0.1476],\n", - " u'56d709ef0d65d21400198307': [u'super bowl xx', 0.1335],\n", - " u'56d70adc0d65d21400198311': [u'justin tucker', 0.9288],\n", - " u'56d70adc0d65d21400198312': [u'natural grass stadiums', 0.1744],\n", - " u'56d70adc0d65d21400198313': [u'super bowl', 0.2015],\n", - " u'56d70adc0d65d21400198314': [u'cleats', 0.7205],\n", - " u'56d70ccc0d65d2140019831d': [u'white', 0.1783],\n", - " u'56d70ccc0d65d2140019831f': [u'white', 0.608],\n", - " u'56d70d240d65d21400198326': [u'stanford university', 0.4228],\n", - " u'56d70d240d65d21400198327': [u'san jose marriott', 0.3928],\n", - " u'56d70d240d65d21400198328': [u'stanford university', 0.7317],\n", - " u'56d70d240d65d21400198329': [u'san jose marriott', 0.4306],\n", - " u'56d70daa0d65d21400198332': [u'june 4, 2014', 0.9524],\n", - " u'56d70daa0d65d21400198333': [u'super bowl li', 0.1502],\n", - " u'56d70daa0d65d21400198334': [u'super bowl 50', 0.1344],\n", - " u'56d70daa0d65d21400198335': [u'colored', 0.2711],\n", - " u'56d70daa0d65d21400198336': [u'june 4, 2014', 0.6931],\n", - " u'56d70e570d65d2140019833c': [u'homecoming', 0.6975],\n", - " u'56d70e570d65d2140019833e': [u'gold footballs', 0.9819],\n", - " u'56d70e570d65d2140019833f': [u'50-yard', 0.3017],\n", - " u'56d7145c0d65d2140019834c': [u'moscone center in san francisco', 0.4041],\n", - " u'56d7145c0d65d2140019834d': [u'super bowl city', 0.3667],\n", - " u'56d7145c0d65d2140019834f': [u'ed lee', 0.8045],\n", - " u'56d714cd0d65d21400198356': [u'bellomy field at santa clara university',\n", - " 0.1399],\n", - " u'56d714cd0d65d21400198358': [u'city council', 0.2915],\n", - " u'56d714cd0d65d21400198359': [u'city council', 0.521],\n", - " u'56d71bd80d65d2140019835e': [u'tuesday afternoon', 0.236],\n", - " u'56d71bd80d65d21400198360': [u'super bowl', 0.5155],\n", - " u'56d71bd80d65d21400198361': [u'sap center in san jose', 0.0378],\n", - " u'56d71bd80d65d21400198362': [u'golden gate bridge', 0.5181],\n", - " u'56d71cb60d65d21400198368': [u'super bowl program', 0.3421],\n", - " u'56d71d150d65d2140019836e': [u'super bowl ever', 0.573],\n", - " u'56d71d150d65d2140019836f': [u'25 percent', 0.6788],\n", - " u'56d71d150d65d21400198370': [u'the 50 fund', 0.2745],\n", - " u'56d71d9d0d65d21400198374': [u'vince lombardi trophy', 0.614],\n", - " u'56d71d9d0d65d21400198377': [u'tiffany & co', 0.9186],\n", - " u'56d71ed50d65d2140019837c': [u'cbs', 0.7906],\n", - " u'56d71ed50d65d2140019837d': [u'three', 0.8812],\n", - " u'56d71fc00d65d21400198386': [u'espn deportes', 0.6344],\n", - " u'56d71fc00d65d21400198389': [u'john sutcliffe', 0.9921],\n", - " u'56d7205e0d65d21400198391': [u'verizon', 0.628],\n", - " u'56d7205e0d65d21400198392': [u'smartphones', 0.3024],\n", - " u'56d720c70d65d21400198396': [u'stephen colbert', 0.7311],\n", - " u'56d720c70d65d21400198397': [u'cbs', 0.0283],\n", - " u'56d720c70d65d21400198398': [u'james corden', 0.0719],\n", - " u'56d721af0d65d2140019839e': [u'nintendo and the pok\\xe9mon company', 0.3748],\n", - " u'56d721af0d65d2140019839f': [u'anheuser-busch inbev', 0.4391],\n", - " u'56d721af0d65d214001983a0': [u'inbev', 0.0745],\n", - " u'56d722330d65d214001983a6': [u'quickbooks', 0.9998],\n", - " u'56d722330d65d214001983a7': [u'death wish coffee', 0.4738],\n", - " u'56d722330d65d214001983a8': [u'nine', 0.9207],\n", - " u'56d723560d65d214001983ac': [u'trailers', 0.4853],\n", - " u'56d723560d65d214001983ad': [u'walt disney studios', 0.1136],\n", - " u'56d723560d65d214001983ae': [u'jason bourne', 0.5854],\n", - " u'56d723560d65d214001983af': [u'civil war', 0.2602],\n", - " u'56d723ad0d65d214001983b4': [u'westwood one', 0.9099],\n", - " u'56d723ad0d65d214001983b5': [u'kevin harlan', 0.7177],\n", - " u'56d723ad0d65d214001983b8': [u'jim gray', 0.9752],\n", - " u'56d724620d65d214001983be': [u'dave logan', 0.6026],\n", - " u'56d724620d65d214001983bf': [u'ed mccaffrey', 0.2497],\n", - " u'56d724620d65d214001983c0': [u'wbt', 0.2665],\n", - " u'56d724620d65d214001983c1': [u'mick mixon', 0.7738],\n", - " u'56d724ea0d65d214001983c8': [u'the bbc', 0.2464],\n", - " u'56d724ea0d65d214001983c9': [u'greg brady', 0.9876],\n", - " u'56d7251d0d65d214001983cc': [u'39', 0.7445],\n", - " u'56d7251d0d65d214001983cd': [u'peyton manning', 0.8198],\n", - " u'56d7251d0d65d214001983cf': [u'super bowl xii', 0.2996],\n", - " u'56d7253b0d65d214001983d4': [u'lady gaga', 0.9609],\n", - " u'56d7253b0d65d214001983d5': [u'marlee matlin', 0.7268],\n", - " u'56d725790d65d214001983d8': [u'bruno mars', 0.8881],\n", - " u'56d725790d65d214001983da': [u'pepsi', 0.4033],\n", - " u'56d725790d65d214001983db': [u'hymn for the weekend', 0.3955],\n", - " u'56d7261e0d65d214001983e0': [u'41-yard line', 0.3296],\n", - " u'56d7261e0d65d214001983e1': [u'ealy', 0.9543],\n", - " u'56d7261e0d65d214001983e2': [u'carolina on the 50-yard line', 0.2016],\n", - " u'56d7261e0d65d214001983e4': [u'punts', 0.664],\n", - " u'56d726b60d65d214001983ea': [u'miller', 0.5379],\n", - " u'56d726b60d65d214001983eb': [u'ward', 0.875],\n", - " u'56d726b60d65d214001983ec': [u'dove', 0.2568],\n", - " u'56d726b60d65d214001983ed': [u'cornerback josh norman', 0.7347],\n", - " u'56d726b60d65d214001983ee': [u'3:08', 0.6562],\n", - " u'56d7270c0d65d214001983f4': [u'three', 0.0447],\n", - " u'56d7270c0d65d214001983f5': [u'three', 0.0374],\n", - " u'56d7270c0d65d214001983f6': [u'anderson', 0.3079],\n", - " u'56d7270c0d65d214001983f7': [u'sanders', 0.4018],\n", - " u'56d7270c0d65d214001983f8': [u'thomas davis', 0.8515],\n", - " u'56d7277c0d65d214001983fe': [u'three', 0.0482],\n", - " u'56d7277c0d65d214001983ff': [u'11', 0.2668],\n", - " u'56d7277c0d65d21400198401': [u'kony ealy', 0.3351],\n", - " u'56d7277c0d65d21400198402': [u'a player', 0.0879],\n", - " u'56d7282f0d65d21400198408': [u'ted ginn jr.', 0.5105],\n", - " u'56d7282f0d65d21400198409': [u'the uprights', 0.563],\n", - " u'56d7282f0d65d2140019840a': [u't. j', 0.5156],\n", - " u'56d7282f0d65d2140019840c': [u'trevathan', 0.4633],\n", - " u'56d728a10d65d21400198413': [u'20', 0.8394],\n", - " u'56d728a10d65d21400198414': [u'linebacker shaq thompson', 0.8978],\n", - " u'56d728a10d65d21400198415': [u'thomas davis', 0.3882],\n", - " u'56d728a10d65d21400198416': [u'a deficit', 0.5947],\n", - " u'56d728e00d65d2140019841c': [u'jerricho cotchery', 0.3456],\n", - " u'56d728e00d65d2140019841d': [u'mike carey', 0.1975],\n", - " u'56d728e00d65d2140019841e': [u'linebacker von miller', 0.5659],\n", - " u'56d728e00d65d2140019841f': [u'malik jackson', 0.5397],\n", - " u'56d728e00d65d21400198420': [u'1993', 0.7016],\n", - " u'56d729180d65d21400198426': [u'25', 0.2598],\n", - " u'56d729180d65d21400198427': [u'jonathan stewart', 0.8745],\n", - " u'56d729180d65d21400198428': [u'10\\u20137', 0.2345],\n", - " u'56d729180d65d21400198429': [u'mario addison', 0.3584],\n", - " u'56d729180d65d2140019842a': [u'25', 0.2279],\n", - " u'56d729ec0d65d21400198430': [u'fullback mike tolbert', 0.2322],\n", - " u'56d729ec0d65d21400198431': [u'kony ealy', 0.8278],\n", - " u'56d729ec0d65d21400198432': [u'19', 0.5344],\n", - " u'56d729ec0d65d21400198434': [u'kony ealy', 0.1866],\n", - " u'56d9895ddc89441400fdb50e': [u'super bowl 50', 0.3315],\n", - " u'56d9895ddc89441400fdb510': [u'denver broncos', 0.212],\n", - " u'56d98a59dc89441400fdb52a': [u'quarterback cam newton', 0.7185],\n", - " u'56d98a59dc89441400fdb52b': [u'arizona cardinals 49\\u201315', 0.2807],\n", - " u'56d98a59dc89441400fdb52e': [u'1995', 0.5341],\n", - " u'56d98b33dc89441400fdb53b': [u'seven', 0.7628],\n", - " u'56d98b33dc89441400fdb53c': [u'seven', 0.1438],\n", - " u'56d98b33dc89441400fdb53d': [u'denver linebacker von miller', 0.6017],\n", - " u'56d98b33dc89441400fdb53e': [u'2\\xbd', 0.1185],\n", - " u'56d98c53dc89441400fdb544': [u'$5 million', 0.8534],\n", - " u'56d98c53dc89441400fdb545': [u'bruno mars', 0.1966],\n", - " u'56d98c53dc89441400fdb546': [u'beyonc\\xe9 and bruno mars', 0.5835],\n", - " u'56d98c53dc89441400fdb548': [u'coldplay with special guest performers beyonc\\xe9 and bruno mars',\n", - " 0.2825],\n", - " u'56d98d0adc89441400fdb54e': [u'spectacular', 0.7234],\n", - " u'56d98d0adc89441400fdb54f': [u'2012', 0.9794],\n", - " u'56d98db6dc89441400fdb552': [u\"miami's sun life stadium\", 0.3372],\n", - " u'56d98db6dc89441400fdb553': [u'superdome', 0.1174],\n", - " u'56d98db6dc89441400fdb554': [u\"miami's sun life stadium\", 0.2383],\n", - " u'56d98f0ddc89441400fdb558': [u'october 16, 2012', 0.9667],\n", - " u'56d98f0ddc89441400fdb559': [u'two', 0.5726],\n", - " u'56d98f0ddc89441400fdb55a': [u'new orleans', 0.6395],\n", - " u'56d98f0ddc89441400fdb55b': [u'1985', 0.4564],\n", - " u'56d98f0ddc89441400fdb55c': [u'florida legislature', 0.4665],\n", - " u'56d98fbfdc89441400fdb562': [u'1985', 0.4185],\n", - " u'56d98fbfdc89441400fdb563': [u'may 21, 2013', 0.8289],\n", - " u'56d98fbfdc89441400fdb564': [u'$1.2 billion', 0.6724],\n", - " u'56d98fbfdc89441400fdb565': [u'1985', 0.3408],\n", - " u'56d99179dc89441400fdb56c': [u'ten', 0.2895],\n", - " u'56d99179dc89441400fdb56d': [u'six', 0.2438],\n", - " u'56d99179dc89441400fdb570': [u'john fox', 0.9856],\n", - " u'56d9943fdc89441400fdb576': [u'carolina panthers', 0.7621],\n", - " u'56d9943fdc89441400fdb577': [u'1978', 0.9153],\n", - " u'56d9943fdc89441400fdb578': [u'nfc team', 0.4354],\n", - " u'56d9943fdc89441400fdb57a': [u'ten', 0.8749],\n", - " u'56d997cddc89441400fdb586': [u'500', 0.7806],\n", - " u'56d997cddc89441400fdb587': [u'3,837', 0.0442],\n", - " u'56d997cddc89441400fdb588': [u'10', 0.0315],\n", - " u'56d997cddc89441400fdb589': [u'99.4', 0.9151],\n", - " u'56d997cddc89441400fdb58a': [u'39', 0.5458],\n", - " u'56d9992fdc89441400fdb59c': [u'four', 0.1622],\n", - " u'56d9992fdc89441400fdb59e': [u'kony ealy', 0.2161],\n", - " u'56d9992fdc89441400fdb59f': [u'jared allen', 0.4473],\n", - " u'56d9992fdc89441400fdb5a0': [u'four', 0.0722],\n", - " u'56d99b7bdc89441400fdb5c8': [u'john fox', 0.2206],\n", - " u'56d99b7bdc89441400fdb5c9': [u'indianapolis colts', 0.4259],\n", - " u'56d99b7bdc89441400fdb5ca': [u'gary kubiak', 0.3523],\n", - " u'56d99b7bdc89441400fdb5cb': [u'osweiler', 0.3675],\n", - " u'56d99b7bdc89441400fdb5cc': [u'john fox', 0.7991],\n", - " u'56d99c44dc89441400fdb5d6': [u'67.9', 0.183],\n", - " u'56d99c44dc89441400fdb5d7': [u'17', 0.6962],\n", - " u'56d99c44dc89441400fdb5d8': [u'emmanuel sanders', 0.261],\n", - " u'56d99c44dc89441400fdb5d9': [u'c. j. anderson', 0.3542],\n", - " u'56d99c44dc89441400fdb5da': [u'4.7', 0.508],\n", - " u'56d99da8dc89441400fdb5fd': [u'linebacker von miller', 0.439],\n", - " u'56d99da8dc89441400fdb5ff': [u'linebacker brandon marshall', 0.1805],\n", - " u'56d99da8dc89441400fdb600': [u'derek wolfe and malik jackson', 0.3273],\n", - " u'56d99e78dc89441400fdb616': [u'arizona cardinals', 0.4455],\n", - " u'56d99e78dc89441400fdb618': [u'487 yards', 0.5714],\n", - " u'56d99e78dc89441400fdb619': [u'arizona cardinals', 0.7362],\n", - " u'56d99e78dc89441400fdb61a': [u'487', 0.7628],\n", - " u'56d99f99dc89441400fdb628': [u'super bowl xlix', 0.1415],\n", - " u'56d99f99dc89441400fdb629': [u'new england patriots', 0.1362],\n", - " u'56d99f99dc89441400fdb62c': [u'17', 0.4259],\n", - " u'56d9a026dc89441400fdb632': [u'thomas davis', 0.9016],\n", - " u'56d9a0eadc89441400fdb63e': [u'peyton manning', 0.9846],\n", - " u'56d9a0eadc89441400fdb63f': [u'39', 0.7497],\n", - " u'56d9a0eadc89441400fdb640': [u'john elway', 0.7552],\n", - " u'56d9a199dc89441400fdb647': [u'2011', 0.3433],\n", - " u'56d9a199dc89441400fdb648': [u'26', 0.6374],\n", - " u'56d9a199dc89441400fdb649': [u'13 years and 48 days', 0.2505],\n", - " u'56d9a419dc89441400fdb678': [u'super bowl', 0.1724],\n", - " u'56d9a419dc89441400fdb679': [u'natural grass stadiums, the nfl re-sodded the field',\n", - " 0.1067],\n", - " u'56d9a637dc89441400fdb697': [u'super bowl in super bowl xxxiii', 0.0707],\n", - " u'56d9a637dc89441400fdb698': [u'white', 0.075],\n", - " u'56d9a637dc89441400fdb699': [u'orange', 0.0597],\n", - " u'56d9a637dc89441400fdb69a': [u'silver', 0.1099],\n", - " u'56d9a6b4dc89441400fdb6a0': [u'san jose marriott', 0.1851],\n", - " u'56d9a6b4dc89441400fdb6a1': [u'san jose marriott', 0.3153],\n", - " u'56d9a6b4dc89441400fdb6a2': [u'stanford university', 0.7317],\n", - " u'56d9a6b4dc89441400fdb6a3': [u'santa clara marriott', 0.3427],\n", - " u'56d9a7addc89441400fdb6a8': [u'arabic', 0.1928],\n", - " u'56d9a7addc89441400fdb6ab': [u'reinstated', 0.232],\n", - " u'56d9b01fdc89441400fdb6d4': [u'colored', 0.1227],\n", - " u'56d9b389dc89441400fdb6ec': [u'super bowl city', 0.6595],\n", - " u'56d9b389dc89441400fdb6ed': [u'1 million', 0.4866],\n", - " u'56d9b389dc89441400fdb6ee': [u'san francisco mayor ed lee', 0.6803],\n", - " u'56d9b389dc89441400fdb6ef': [u'$5 million', 0.9289],\n", - " u'56d9b389dc89441400fdb6f0': [u'super bowl city', 0.1644],\n", - " u'56d9b43edc89441400fdb700': [u'$2 million', 0.9277],\n", - " u'56d9b43edc89441400fdb701': [u'city council', 0.1816],\n", - " u'56d9b4ebdc89441400fdb708': [u'opening night', 0.2111],\n", - " u'56d9b4ebdc89441400fdb709': [u'super bowl opening night', 0.1954],\n", - " u'56d9b4ebdc89441400fdb70a': [u'sap center', 0.2514],\n", - " u'56d9b4ebdc89441400fdb70b': [u'golden gate bridge', 0.5753],\n", - " u'56d9b4ebdc89441400fdb70c': [u'february 1, 2016', 0.8979],\n", - " u'56d9b5ffdc89441400fdb720': [u'super bowl program', 0.4281],\n", - " u'56d9b5ffdc89441400fdb721': [u'over $40 million', 0.6042],\n", - " u'56d9b6a3dc89441400fdb72f': [u'25 percent', 0.5351],\n", - " u'56d9b6a3dc89441400fdb730': [u'the 50 fund', 0.2309],\n", - " u'56d9b7dcdc89441400fdb740': [u'cbs', 0.6554],\n", - " u'56d9b7dcdc89441400fdb741': [u'jim nantz and phil simms', 0.2216],\n", - " u'56d9b7dcdc89441400fdb742': [u'tracy wolfson and evan washburn', 0.0493],\n", - " u'56d9b7dcdc89441400fdb743': [u'5k resolution', 0.6302],\n", - " u'56d9b7dcdc89441400fdb744': [u'5k resolution', 0.3965],\n", - " u'56d9ba95dc89441400fdb754': [u'espn deportes', 0.578],\n", - " u'56d9ba95dc89441400fdb757': [u'john sutcliffe', 0.248],\n", - " u'56d9ba95dc89441400fdb758': [u'john sutcliffe', 0.9672],\n", - " u'56d9bc13dc89441400fdb75e': [u'digital streams', 0.5363],\n", - " u'56d9bc13dc89441400fdb75f': [u'verizon wireless', 0.4756],\n", - " u'56d9bc13dc89441400fdb760': [u'watchespn', 0.875],\n", - " u'56d9bdc1dc89441400fdb768': [u'anheuser-busch inbev', 0.3774],\n", - " u'56d9bdc1dc89441400fdb769': [u'inbev', 0.0913],\n", - " u'56d9bdc1dc89441400fdb76a': [u'crash the super bowl', 0.2962],\n", - " u'56d9be16dc89441400fdb770': [u'quickbooks', 0.4407],\n", - " u'56d9be16dc89441400fdb771': [u'quickbooks', 0.1289],\n", - " u'56d9bf70dc89441400fdb77b': [u'teenage mutant ninja turtles', 0.0226],\n", - " u'56d9bf70dc89441400fdb77c': [u'walt disney studios', 0.3591],\n", - " u'56d9bf70dc89441400fdb77d': [u'walt disney studios', 0.1572],\n", - " u'56d9c049dc89441400fdb78e': [u'kevin harlan', 0.5948],\n", - " u'56d9c049dc89441400fdb78f': [u'boomer esiason and dan fouts', 0.4748],\n", - " u'56d9c049dc89441400fdb790': [u'kevin harlan', 0.2678],\n", - " u'56d9c049dc89441400fdb791': [u'anchor the pre-game and halftime coverage',\n", - " 0.5616],\n", - " u'56d9c049dc89441400fdb792': [u'north america', 0.7735],\n", - " u'56d9c203dc89441400fdb7a0': [u'koa ( 850 am) and krfx ( 103.5 fm)', 0.1049],\n", - " u'56d9c203dc89441400fdb7a3': [u'wbt-fm', 0.3026],\n", - " u'56d9c3a6dc89441400fdb7b4': [u'39', 0.5695],\n", - " u'56d9c3a6dc89441400fdb7b5': [u'39', 0.8993],\n", - " u'56d9c3a6dc89441400fdb7b6': [u'bart starr', 0.9347],\n", - " u'56d9c3a6dc89441400fdb7b7': [u'peyton manning', 0.3267],\n", - " u'56d9c3a6dc89441400fdb7b8': [u'super bowl xii', 0.5551],\n", - " u'56d9c455dc89441400fdb7c4': [u'lady gaga', 0.9334],\n", - " u'56d9c455dc89441400fdb7c5': [u'six-time grammy winner', 0.1119],\n", - " u'56d9c455dc89441400fdb7c6': [u'marlee matlin', 0.6344],\n", - " u'56d9c551dc89441400fdb7d0': [u'bruno mars', 0.7604],\n", - " u'56d9c551dc89441400fdb7d1': [u'beyonc\\xe9', 0.1358],\n", - " u'56d9c551dc89441400fdb7d2': [u'hymn for the weekend', 0.5549],\n", - " u'56d9c551dc89441400fdb7d3': [u'bruno mars', 0.7961],\n", - " u'56d9c5e7dc89441400fdb7d8': [u'denver', 0.8392],\n", - " u'56d9c5e7dc89441400fdb7d9': [u'owen daniels', 0.8117],\n", - " u'56d9c5e7dc89441400fdb7da': [u'c. j. anderson', 0.8099],\n", - " u'56d9c5e7dc89441400fdb7db': [u'c. j. anderson', 0.1179],\n", - " u'56d9c5e7dc89441400fdb7dc': [u'a deficit', 0.4916],\n", - " u'56d9c660dc89441400fdb7e7': [u'mike carey', 0.7943],\n", - " u'56d9c660dc89441400fdb7e8': [u'linebacker von miller', 0.8433],\n", - " u'56d9c660dc89441400fdb7e9': [u'malik jackson', 0.938],\n", - " u'56d9c660dc89441400fdb7ea': [u'1993 season', 0.3279],\n", - " u'56d9c6e0dc89441400fdb7f0': [u'jonathan stewart', 0.9616],\n", - " u'56d9c6e0dc89441400fdb7f4': [u'10\\u20137', 0.0445],\n", - " u'56d9c79edc89441400fdb804': [u'kony ealy', 0.794],\n", - " u'56d9c79edc89441400fdb805': [u'safety darian stewart', 0.3945],\n", - " u'56d9c79edc89441400fdb806': [u'kony ealy', 0.9788],\n", - " u'56d9c79edc89441400fdb807': [u'39-yard line with 1:55 left on the clock',\n", - " 0.0247],\n", - " u'56d9c79edc89441400fdb808': [u'demarcus ware', 0.9852],\n", - " u'56d9c92bdc89441400fdb80e': [u'corey brown', 0.2491],\n", - " u'56d9c92bdc89441400fdb80f': [u'26-yard line', 0.461],\n", - " u'56d9c92bdc89441400fdb810': [u'graham gano', 0.9753],\n", - " u'56d9c92bdc89441400fdb811': [u'graham gano', 0.4791],\n", - " u'56d9ca0adc89441400fdb820': [u'ealy', 0.7043],\n", - " u'56d9ca0adc89441400fdb821': [u'50-yard line', 0.2183],\n", - " u'56d9ca0adc89441400fdb822': [u'one', 0.1662],\n", - " u'56d9ca0adc89441400fdb823': [u'three', 0.2059],\n", - " u'56d9cb47dc89441400fdb832': [u'3rd-and-9', 0.352],\n", - " u'56d9cb47dc89441400fdb833': [u'miller', 0.4125],\n", - " u'56d9cb47dc89441400fdb834': [u'ward', 0.7671],\n", - " u'56d9cb47dc89441400fdb835': [u'three', 0.7815],\n", - " u'56d9cb47dc89441400fdb836': [u'bennie fowler', 0.3966],\n", - " u'56d9ccacdc89441400fdb842': [u'sanders', 0.4497],\n", - " u'56d9ccacdc89441400fdb843': [u'anderson', 0.2113],\n", - " u'56d9ccacdc89441400fdb844': [u'four', 0.2331],\n", - " u'56d9ccacdc89441400fdb845': [u'six', 0.0542],\n", - " u'56d9ccacdc89441400fdb846': [u'four', 0.1513],\n", - " u'56d9cd9adc89441400fdb84c': [u'11', 0.2455],\n", - " u'56d9cd9adc89441400fdb84d': [u'11', 0.1422],\n", - " u'56d9cd9adc89441400fdb84e': [u'baltimore ravens', 0.602],\n", - " u'56d9cd9adc89441400fdb84f': [u'kony ealy', 0.4073],\n", - " u'56d9cd9adc89441400fdb850': [u'two', 0.5988]}\n" - ] - } - ], - "source": [ - "preds_prob = json.load(open('predictions_bidaf_prob.json'))\n", - "preds_bidaf = json.load(open('predictions_bidaf_attn.json'))\n", - "\n", - "pprint(preds_prob)" - ] - }, - { - "cell_type": "code", - "execution_count": 100, - "metadata": {}, - "outputs": [], - "source": [ - "def heat(sample,weights,dark=0.7):\n", - " weights = (weights - weights.min())/(weights.max() - weights.min() + 1e-9)\n", - " print(weights)\n", - " html = ''\n", - " fmt = ' {1}'\n", - " for t,w in zip(sample,weights):\n", - " if t!= 'PAD':\n", - " c = int(256*((1.-dark)*(1.-w)+dark))\n", - " html += fmt.format(c,t)\n", - " display(HTML(html))" - ] - }, - { - "cell_type": "code", - "execution_count": 101, - "metadata": {}, - "outputs": [], - "source": [ - "def plot_attention(uuid, context_asis, preds_bidaf):\n", - " #find the corresponding uuid in preds_bidaf\n", - " \n", - " if uuid in preds_bidaf:\n", - " val = preds_bidaf[uuid]\n", - " answer = val[0]\n", - " prob = val[1]\n", - " alphas = val[2]\n", - " \n", - " context = context_asis.decode('ascii', 'ignore')\n", - "\n", - " # The following replacements are suggested in the paper\n", - " # BidAF (Seo et al., 2016)\n", - " context = context.replace(\"''\", '\" ')\n", - " context = context.replace(\"``\", '\" ')\n", - " context = context.lower()\n", - " words = context.split(' ')\n", - " num_words = len(words)\n", - " print(words)\n", - " alphas_small = alphas[:num_words]\n", - " print(sum(alphas_small))\n", - " print(len(words), len(alphas_small))\n", - " array_alpha = np.asarray(alphas_small)\n", - " heat(words, array_alpha)\n", - "\n", - " print(\"Predicted answer: \", answer)\n", - " print(\"Prob\", prob)" - ] - }, - { - "cell_type": "code", - "execution_count": 102, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "from IPython.core.display import display, HTML\n" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[u'manning', u'finished', u'the', u'game', u'13', u'of', u'23', u'for', u'141', u'yards', u'with', u'one', u'interception', u'and', u'zero', u'touchdowns.', u'sanders', u'was', u'his', u'top', u'receiver', u'with', u'six', u'receptions', u'for', u'83', u'yards.', u'anderson', u'was', u'the', u'game', u'leading', u'rusher', u'with', u'90', u'yards', u'and', u'a', u'touchdown,', u'along', u'with', u'four', u'receptions', u'for', u'10', u'yards.', u'miller', u'had', u'six', u'total', u'tackles', u'(five', u'solo),', u'sacks,', u'and', u'two', u'forced', u'fumbles.', u'ware', u'had', u'five', u'total', u'tackles', u'and', u'two', u'sacks.', u'ward', u'had', u'seven', u'total', u'tackles,', u'a', u'fumble', u'recovery,', u'and', u'an', u'interception.', u'mcmanus', u'made', u'all', u'four', u'of', u'his', u'field', u'goals,', u'making', u'him', u'perfect', u'on', u'all', u'11', u'attempts', u'during', u'the', u'post-season.', u'newton', u'completed', u'18', u'of', u'41', u'passes', u'for', u'265', u'yards,', u'with', u'one', u'interception.', u'he', u'was', u'also', u'the', u'team', u'leading', u'rusher', u'with', u'45', u'yards', u'on', u'six', u'carries.', u'brown', u'caught', u'four', u'passes', u'for', u'80', u'yards,', u'while', u'ginn', u'had', u'four', u'receptions', u'for', u'74.', u'ealy', u'was', u'the', u'top', u'defensive', u'performer', u'for', u'carolina', u'with', u'four', u'total', u'tackles,', u'three', u'sacks,', u'a', u'forced', u'fumble,', u'a', u'fumble', u'recovery,', u'and', u'an', u'interception.', u'defensive', u'end', u'charles', u'johnson', u'had', u'four', u'total', u'tackles,', u'a', u'sack,', u'and', u'a', u'forced', u'fumble.', u'linebacker', u'luke', u'kuechly', u'had', u'11', u'total', u'tackles,', u'while', u'thomas', u'davis', u'had', u'seven,', u'despite', u'playing', u'just', u'two', u'weeks', u'after', u'breaking', u'his', u'right', u'arm', u'in', u'the', u'nfc', u'title', u'game']\n", - "0.842608284231\n", - "198 198\n", - "[1.44517366e-02 1.99681535e-03 1.88156541e-03 1.33650430e-02\n", - " 5.77653478e-02 3.29881634e-05 1.78608160e-02 5.54641474e-05\n", - " 1.05704869e-02 2.48094610e-05 2.83095274e-05 9.86125800e-04\n", - " 4.88344421e-05 5.98972210e-06 3.94877988e-04 1.42904938e-03\n", - " 4.60488780e-04 7.28109224e-03 2.04779429e-04 4.06624606e-04\n", - " 3.16417222e-03 8.52899551e-04 3.49063666e-04 3.13428354e-01\n", - " 1.00751706e-04 4.34728758e-05 1.39340605e-01 1.24481408e-04\n", - " 2.17493636e-04 4.02758402e-02 1.59481672e-04 9.56294148e-04\n", - " 1.69143468e-03 2.95926382e-05 9.67839975e-04 1.21881228e-03\n", - " 2.17337957e-04 5.23829819e-02 1.32847199e-04 6.87848485e-06\n", - " 6.48533441e-04 3.13818269e-03 3.56575437e-04 1.88733767e-03\n", - " 1.75890179e-04 1.51963166e-01 9.94118760e-05 5.86505082e-05\n", - " 8.95275392e-02 1.56141156e-04 1.27610845e-03 8.60294161e-02\n", - " 7.85272828e-04 1.54591347e-01 4.86877946e-05 6.84476194e-04\n", - " 2.30472487e-04 5.80291222e-02 3.07706266e-04 9.10679042e-06\n", - " 1.74263226e-05 8.33302085e-02 4.46653184e-05 6.20843567e-06\n", - " 3.59719221e-05 1.96426144e-02 5.02371232e-05 4.21537714e-04\n", - " 2.51143996e-03 1.24495752e-01 1.03928602e-03 4.72434685e-01\n", - " 5.48459263e-05 7.61261131e-04 9.65675005e-06 1.48190326e-01\n", - " 1.82326261e-04 2.34487734e-03 1.02237619e-01 9.09711713e-04\n", - " 4.77928150e-01 3.91919197e-05 3.75458133e-04 3.59633329e-06\n", - " 3.56944993e-04 3.71652356e-04 1.61518594e-05 4.37946265e-06\n", - " 2.57012158e-05 2.93146800e-04 9.08518453e-04 1.47008060e-03\n", - " 8.29385135e-03 1.20388540e-04 5.55379463e-03 7.82236931e-02\n", - " 2.03917892e-06 1.86978347e-05 4.27956368e-05 1.79815951e-05\n", - " 2.54146312e-05 5.99653703e-04 1.86349236e-04 5.88857686e-04\n", - " 2.67675861e-04 1.77594116e-03 1.57850134e-02 1.38921713e-05\n", - " 9.01774404e-05 6.59086825e-05 2.70204993e-04 3.36333153e-03\n", - " 9.06638424e-02 1.70420479e-02 9.99999994e-01 3.58978832e-05\n", - " 1.12122864e-01 1.60544450e-04 1.39878475e-04 2.52044715e-01\n", - " 2.80985842e-04 2.28462642e-05 4.85183604e-04 2.79017425e-02\n", - " 5.10840560e-04 1.14584470e-03 6.54447060e-03 2.76751522e-04\n", - " 4.30478063e-05 6.86213430e-04 1.19707917e-03 6.57979954e-05\n", - " 6.28506215e-04 1.00117109e-03 5.26449041e-05 3.30194895e-02\n", - " 8.11033251e-05 2.52462882e-04 3.81822502e-02 2.20900541e-05\n", - " 6.62400945e-04 3.26141255e-02 2.02642350e-03 1.11297551e-01\n", - " 4.66753371e-05 7.14914201e-05 4.54653656e-02 3.99103803e-04\n", - " 3.05401421e-05 4.20250139e-03 7.39035064e-02 2.90290659e-04\n", - " 1.50311172e-01 9.16886220e-05 6.91473125e-05 2.31779056e-01\n", - " 2.26197154e-05 1.62206396e-03 2.53253823e-05 5.43692014e-05\n", - " 1.96933832e-04 4.37890174e-05 4.05649042e-05 5.11659477e-06\n", - " 2.82500169e-04 1.93896180e-05 8.05350552e-03 6.58940816e-06\n", - " 1.87585090e-05 1.79033816e-06 8.96922703e-03 3.01911667e-06\n", - " 0.00000000e+00 4.52684692e-05 6.18150166e-06 9.79980260e-06\n", - " 1.67452893e-07 2.73971892e-05 2.13282680e-05 6.33687024e-07\n", - " 1.17878574e-07 3.15305137e-06 6.63426133e-05 1.22429308e-04\n", - " 1.15124290e-04 3.79280344e-03 1.35739015e-04 1.28724701e-02\n", - " 2.42174519e-04 2.04150423e-04 1.11585948e-02 4.16329356e-06\n", - " 3.21716914e-05 5.34296748e-07 1.25284605e-04 7.38324370e-05\n", - " 1.00608445e-06 1.17412780e-06]\n" - ] - }, - { - "data": { - "text/html": [ - " manning finished the game 13 of 23 for 141 yards with one interception and zero touchdowns. sanders was his top receiver with six receptions for 83 yards. anderson was the game leading rusher with 90 yards and a touchdown, along with four receptions for 10 yards. miller had six total tackles (five solo), sacks, and two forced fumbles. ware had five total tackles and two sacks. ward had seven total tackles, a fumble recovery, and an interception. mcmanus made all four of his field goals, making him perfect on all 11 attempts during the post-season. newton completed 18 of 41 passes for 265 yards, with one interception. he was also the team leading rusher with 45 yards on six carries. brown caught four passes for 80 yards, while ginn had four receptions for 74. ealy was the top defensive performer for carolina with four total tackles, three sacks, a forced fumble, a fumble recovery, and an interception. defensive end charles johnson had four total tackles, a sack, and a forced fumble. linebacker luke kuechly had 11 total tackles, while thomas davis had seven, despite playing just two weeks after breaking his right arm in the nfc title game" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Predicted answer: 18\n", - "Prob 0.0212\n" - ] - } - ], - "source": [ - "\n", - "context = 'Manning finished the game 13 of 23 for 141 yards with one interception and zero touchdowns. Sanders was his top receiver with six receptions for 83 yards. Anderson was the game leading rusher with 90 yards and a touchdown, along with four receptions for 10 yards. Miller had six total tackles (five solo), sacks, and two forced fumbles. Ware had five total tackles and two sacks. Ward had seven total tackles, a fumble recovery, and an interception. McManus made all four of his field goals, making him perfect on all 11 attempts during the post-season. Newton completed 18 of 41 passes for 265 yards, with one interception. He was also the team leading rusher with 45 yards on six carries. Brown caught four passes for 80 yards, while Ginn had four receptions for 74. Ealy was the top defensive performer for Carolina with four total tackles, three sacks, a forced fumble, a fumble recovery, and an interception. Defensive End Charles Johnson had four total tackles, a sack, and a forced fumble. Linebacker Luke Kuechly had 11 total tackles, while Thomas Davis had seven, despite playing just two weeks after breaking his right arm in the NFC title game'\n", - "\n", - "uuid = '56d9ccacdc89441400fdb845'\n", - "plot_attention(uuid,context,preds_bidaf)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/code/official_eval_helper_best.py b/code/official_eval_helper_best.py deleted file mode 100644 index 627c1cd..0000000 --- a/code/official_eval_helper_best.py +++ /dev/null @@ -1,281 +0,0 @@ -# Copyright 2018 Stanford University -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This code is required for "official_eval" mode in main.py -It provides functions to read a SQuAD json file, use the model to get predicted answers, -and write those answers to another JSON file.""" - -from __future__ import absolute_import -from __future__ import division - -import os -from tqdm import tqdm -import numpy as np -from six.moves import xrange -from nltk.tokenize.moses import MosesDetokenizer - -from preprocessing.squad_preprocess import data_from_json, tokenize -from vocab import UNK_ID, PAD_ID -from data_batcher import padded, Batch - - - -def readnext(x): - """x is a list""" - if len(x) == 0: - return False - else: - return x.pop(0) - - - -def refill_batches(batches, word2id, qn_uuid_data, context_token_data, qn_token_data, batch_size, context_len, question_len): - """ - This is similar to refill_batches in data_batcher.py, but: - (1) instead of reading from (preprocessed) datafiles, it reads from the provided lists - (2) it only puts the context and question information in the batches (not the answer information) - (3) it also gets UUID information and puts it in the batches - - Inputs: - batches: list to be refilled - qn_uuid_data: list of strings that are unique ids - context_token_data, qn_token_data: list of lists of strings (no UNKs, no padding) - batch_size: int. size of batches to make - context_len, question_len: ints. max sizes of context and question. Anything longer is truncated. - - Makes batches that contain: - uuids_batch, context_tokens_batch, context_ids_batch, qn_ids_batch: all lists length batch_size - """ - examples = [] - - # Get next example - qn_uuid, context_tokens, qn_tokens = readnext(qn_uuid_data), readnext(context_token_data), readnext(qn_token_data) - - while qn_uuid and context_tokens and qn_tokens: - - # Convert context_tokens and qn_tokens to context_ids and qn_ids - context_ids = [word2id.get(w, UNK_ID) for w in context_tokens] - qn_ids = [word2id.get(w, UNK_ID) for w in qn_tokens] - - # Truncate context_ids and qn_ids - # Note: truncating context_ids may truncate the correct answer, meaning that it's impossible for your model to get the correct answer on this example! - if len(qn_ids) > question_len: - qn_ids = qn_ids[:question_len] - if len(context_ids) > context_len: - context_ids = context_ids[:context_len] - - # Add to list of examples - examples.append((qn_uuid, context_tokens, context_ids, qn_ids)) - - # Stop if you've got a batch - if len(examples) == batch_size: - break - - # Get next example - qn_uuid, context_tokens, qn_tokens = readnext(qn_uuid_data), readnext(context_token_data), readnext(qn_token_data) - - # Make into batches - for batch_start in xrange(0, len(examples), batch_size): - uuids_batch, context_tokens_batch, context_ids_batch, qn_ids_batch = zip(*examples[batch_start:batch_start + batch_size]) - - batches.append((uuids_batch, context_tokens_batch, context_ids_batch, qn_ids_batch)) - - return - - - -def get_batch_generator(word2id, qn_uuid_data, context_token_data, qn_token_data, batch_size, context_len, question_len): - """ - This is similar to get_batch_generator in data_batcher.py, but with some - differences (see explanation in refill_batches). - - Inputs: - word2id: dictionary mapping word (string) to word id (int) - qn_uuid_data: list of strings that are unique ids - context_token_data, qn_token_data: list of lists of strings (no UNKs, no padding) - batch_size: int. size of batches to make - context_len, question_len: ints. max sizes of context and question. Anything longer is truncated. - - Yields: - Batch objects, but they only contain context and question information (no answer information) - """ - batches = [] - - while True: - if len(batches) == 0: - refill_batches(batches, word2id, qn_uuid_data, context_token_data, qn_token_data, batch_size, context_len, question_len) - if len(batches) == 0: - break - - # Get next batch. These are all lists length batch_size - (uuids, context_tokens, context_ids, qn_ids) = batches.pop(0) - - # Pad context_ids and qn_ids - qn_ids = padded(qn_ids, question_len) # pad questions to length question_len - context_ids = padded(context_ids, context_len) # pad contexts to length context_len - - # Make qn_ids into a np array and create qn_mask - qn_ids = np.array(qn_ids) - qn_mask = (qn_ids != PAD_ID).astype(np.int32) - - # Make context_ids into a np array and create context_mask - context_ids = np.array(context_ids) - context_mask = (context_ids != PAD_ID).astype(np.int32) - - # Make into a Batch object - batch = Batch(context_ids, context_mask, context_tokens, qn_ids, qn_mask, qn_tokens=None, ans_span=None, ans_tokens=None, uuids=uuids) - - yield batch - - return - - -def preprocess_dataset(dataset): - """ - Note: this is similar to squad_preprocess.preprocess_and_write, but: - (1) We only extract the context and question information from the JSON file. - We don't extract answer information. This makes this function much simpler - than squad_preprocess.preprocess_and_write, because we don't have to convert - the character spans to word spans. This also means that we don't have to - discard any examples due to tokenization problems. - - Input: - dataset: data read from SQuAD JSON file - - Returns: - qn_uuid_data, context_token_data, qn_token_data: lists of uuids, tokenized context and tokenized questions - """ - qn_uuid_data = [] - context_token_data = [] - qn_token_data = [] - - for articles_id in tqdm(range(len(dataset['data'])), desc="Preprocessing data"): - article_paragraphs = dataset['data'][articles_id]['paragraphs'] - for pid in range(len(article_paragraphs)): - - context = unicode(article_paragraphs[pid]['context']) # string - - # The following replacements are suggested in the paper - # BidAF (Seo et al., 2016) - context = context.replace("''", '" ') - context = context.replace("``", '" ') - - context_tokens = tokenize(context) # list of strings (lowercase) - context = context.lower() - - qas = article_paragraphs[pid]['qas'] # list of questions - - # for each question - for qn in qas: - - # read the question text and tokenize - question = unicode(qn['question']) # string - question_tokens = tokenize(question) # list of strings - - # also get the question_uuid - question_uuid = qn['id'] - - # Append to data lists - qn_uuid_data.append(question_uuid) - context_token_data.append(context_tokens) - qn_token_data.append(question_tokens) - - return qn_uuid_data, context_token_data, qn_token_data - - -def get_json_data(data_filename): - """ - Read the contexts and questions from a .json file (like dev-v1.1.json) - - Returns: - qn_uuid_data: list (length equal to dev set size) of unicode strings like '56be4db0acb8001400a502ec' - context_token_data, qn_token_data: lists (length equal to dev set size) of lists of strings (no UNKs, unpadded) - """ - # Check the data file exists - if not os.path.exists(data_filename): - raise Exception("JSON input file does not exist: %s" % data_filename) - - # Read the json file - print "Reading data from %s..." % data_filename - data = data_from_json(data_filename) - - # Get the tokenized contexts and questions, and unique question identifiers - print "Preprocessing data from %s..." % data_filename - qn_uuid_data, context_token_data, qn_token_data = preprocess_dataset(data) - - data_size = len(qn_uuid_data) - assert len(context_token_data) == data_size - assert len(qn_token_data) == data_size - print "Finished preprocessing. Got %i examples from %s" % (data_size, data_filename) - - return qn_uuid_data, context_token_data, qn_token_data - - -def generate_answers(session, model, word2id, qn_uuid_data, context_token_data, qn_token_data): - """ - Given a model, and a set of (context, question) pairs, each with a unique ID, - use the model to generate an answer for each pair, and return a dictionary mapping - each unique ID to the generated answer. - - Inputs: - session: TensorFlow session - model: QAModel - word2id: dictionary mapping word (string) to word id (int) - qn_uuid_data, context_token_data, qn_token_data: lists - - Outputs: - uuid2ans: dictionary mapping uuid (string) to predicted answer (string; detokenized) - """ - uuid2ans = {} # maps uuid to string containing predicted answer - data_size = len(qn_uuid_data) - num_batches = ((data_size-1) / model.FLAGS.batch_size) + 1 - batch_num = 0 - detokenizer = MosesDetokenizer() - - print "Generating answers..." - - for batch in get_batch_generator(word2id, qn_uuid_data, context_token_data, qn_token_data, model.FLAGS.batch_size, model.FLAGS.context_len, model.FLAGS.question_len): - - # Get the predicted spans - pred_start_batch, pred_end_batch = model.get_start_end_pos(session, batch) - - # Convert pred_start_batch and pred_end_batch to lists length batch_size - pred_start_batch = pred_start_batch.tolist() - pred_end_batch = pred_end_batch.tolist() - - # For each example in the batch: - for ex_idx, (pred_start, pred_end) in enumerate(zip(pred_start_batch, pred_end_batch)): - - # Original context tokens (no UNKs or padding) for this example - context_tokens = batch.context_tokens[ex_idx] # list of strings - - # Check the predicted span is in range - assert pred_start in range(len(context_tokens)) - assert pred_end in range(len(context_tokens)) - - # Predicted answer tokens - pred_ans_tokens = context_tokens[pred_start : pred_end +1] # list of strings - - # Detokenize and add to dict - uuid = batch.uuids[ex_idx] - uuid2ans[uuid] = detokenizer.detokenize(pred_ans_tokens, return_str=True) - - batch_num += 1 - - if batch_num % 10 == 0: - print "Generated answers for %i/%i batches = %.2f%%" % (batch_num, num_batches, batch_num*100.0/num_batches) - - print "Finished generating answers for dataset." - - return uuid2ans diff --git a/code/qa_model_best.py b/code/qa_model_best.py deleted file mode 100644 index 0b5c28e..0000000 --- a/code/qa_model_best.py +++ /dev/null @@ -1,860 +0,0 @@ -# Copyright 2018 Stanford University -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""This file defines the top-level model""" - -from __future__ import absolute_import -from __future__ import division - -import time -import logging -import os -import sys - -import numpy as np -import tensorflow as tf -from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.ops import embedding_ops - -from evaluate import exact_match_score, f1_score -from data_batcher import get_batch_generator -from pretty_print import print_example -# from modules import RNNEncoder, SimpleSoftmaxLayer, BasicAttn, Basic_Additive_Attn -from modules import * - -logging.basicConfig(level=logging.INFO) - - -class QAModel(object): - """Top-level Question Answering module""" - - def __init__(self, FLAGS, id2word, word2id, emb_matrix): - """ - Initializes the QA model. - - Inputs: - FLAGS: the flags passed in from main.py - id2word: dictionary mapping word idx (int) to word (string) - word2id: dictionary mapping word (string) to word idx (int) - emb_matrix: numpy array shape (400002, embedding_size) containing pre-traing GloVe embeddings - """ - print "Initializing the QAModel..." - self.FLAGS = FLAGS - self.id2word = id2word - self.word2id = word2id - _, _, num_chars = self.create_char_dicts() - self.char_vocab = num_chars - - # Add all parts of the graph - with tf.variable_scope("QAModel", initializer=tf.contrib.layers.variance_scaling_initializer(factor=1.0, uniform=True)): - self.add_placeholders() - self.add_embedding_layer(emb_matrix) - self.build_graph() - self.add_loss() - - # Define trainable parameters, gradient, gradient norm, and clip by gradient norm - params = tf.trainable_variables() - gradients = tf.gradients(self.loss, params) - self.gradient_norm = tf.global_norm(gradients) - clipped_gradients, _ = tf.clip_by_global_norm(gradients, FLAGS.max_gradient_norm) - self.param_norm = tf.global_norm(params) - - # Define optimizer and updates - # (updates is what you need to fetch in session.run to do a gradient update) - self.global_step = tf.Variable(0, name="global_step", trainable=False) - opt = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate) # you can try other optimizers - self.updates = opt.apply_gradients(zip(clipped_gradients, params), global_step=self.global_step) - - # Define savers (for checkpointing) and summaries (for tensorboard) - self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.keep) - self.bestmodel_saver = tf.train.Saver(tf.global_variables(), max_to_keep=1) - self.summaries = tf.summary.merge_all() - - - def add_placeholders(self): - """ - Add placeholders to the graph. Placeholders are used to feed in inputs. - """ - # Add placeholders for inputs. - # These are all batch-first: the None corresponds to batch_size and - # allows you to run the same model with variable batch_size - self.context_ids = tf.placeholder(tf.int32, shape=[None, self.FLAGS.context_len]) - self.context_mask = tf.placeholder(tf.int32, shape=[None, self.FLAGS.context_len]) - self.qn_ids = tf.placeholder(tf.int32, shape=[None, self.FLAGS.question_len]) - self.qn_mask = tf.placeholder(tf.int32, shape=[None, self.FLAGS.question_len]) - self.ans_span = tf.placeholder(tf.int32, shape=[None, 2]) - - # Add a placeholder to feed in the keep probability (for dropout). - # This is necessary so that we can instruct the model to use dropout when training, but not when testing - self.keep_prob = tf.placeholder_with_default(1.0, shape=()) - - ## For Char CNN - self.char_ids_context = tf.placeholder(tf.int32, shape=[None, self.FLAGS.context_len, self.FLAGS.word_max_len]) - self.char_ids_qn = tf.placeholder(tf.int32, shape=[None, self.FLAGS.question_len, self.FLAGS.word_max_len]) - - - def add_embedding_layer(self, emb_matrix): - """ - Adds word embedding layer to the graph. - - Inputs: - emb_matrix: shape (400002, embedding_size). - The GloVe vectors, plus vectors for PAD and UNK. - """ - with vs.variable_scope("embeddings"): - - # Note: the embedding matrix is a tf.constant which means it's not a trainable parameter - embedding_matrix = tf.constant(emb_matrix, dtype=tf.float32, name="emb_matrix") # shape (400002, embedding_size) - - # Get the word embeddings for the context and question, - # using the placeholders self.context_ids and self.qn_ids - self.context_embs = embedding_ops.embedding_lookup(embedding_matrix, self.context_ids) # shape (batch_size, context_len, embedding_size) - self.qn_embs = embedding_ops.embedding_lookup(embedding_matrix, self.qn_ids) # shape (batch_size, question_len, embedding_size) - - def add_char_embeddings(self): - """ - Adds char embedding layer to the graph. - - """ - - def conv1d(input_, output_size, width, stride, scope_name): - ''' - :param input_: A tensor of embedded tokens with shape [batch_size,max_length,embedding_size] - :param output_size: The number of feature maps we'd like to calculate - :param width: The filter width - :param stride: The stride - :return: A tensor of the concolved input with shape [batch_size,max_length,output_size] - ''' - inputSize = input_.get_shape()[ - -1] # How many channels on the input (The size of our embedding for instance) - - # This is the kicker where we make our text an image of height 1 - input_ = tf.expand_dims(input_, axis=1) # Change the shape to [batch_size,1,max_length,output_size] - - # Make sure the height of the filter is 1 - with tf.variable_scope(scope_name, reuse=tf.AUTO_REUSE): - filter_ = tf.get_variable("conv_filter", shape=[1, width, inputSize, output_size]) - - # Run the convolution as if this were an image - convolved = tf.nn.conv2d(input_, filter=filter_, strides=[1, 1, stride, 1], padding="VALID") - # Remove the extra dimension, eg make the shape [batch_size,max_length,output_size] - result = tf.squeeze(convolved, axis=1) - return result - - with vs.variable_scope("char_embeddings"): - - # Note: the embedding matrix is a tf.constant which means it's not a trainable parameter - char_emb_matrix = tf.Variable(tf.random_uniform((self.char_vocab, self.FLAGS.char_embedding_size), -1, 1)) #is trainable - - print("Shape context placeholder", self.char_ids_context.shape) - print("Shape qn placeholder", self.char_ids_qn.shape) - - self.context_char_embs = embedding_ops.embedding_lookup(char_emb_matrix, tf.reshape(self.char_ids_context, shape=(-1, self.FLAGS.word_max_len))) # shape (-1, word_max_len, char_embedding_size) - - ##reshape to 3d tensor - compress dimensions we don't want to convolve on - self.context_char_embs = tf.reshape(self.context_char_embs, shape=( - -1, self.FLAGS.word_max_len, self.FLAGS.char_embedding_size)) # shape = batch_size*context_len, word_max_len, char_embedding_size - - print("Shape context embs before conv", self.context_char_embs.shape) - - ## Repeat for question embeddings - again reshape to 3D tensor - self.qn_char_embs = embedding_ops.embedding_lookup(char_emb_matrix, tf.reshape(self.char_ids_qn, shape=(-1, self.FLAGS.word_max_len))) - self.qn_char_embs = tf.reshape(self.qn_char_embs, shape=(-1, self.FLAGS.word_max_len, self.FLAGS.char_embedding_size)) - - - print("Shape qn embs before conv", self.qn_char_embs.shape) - - ## Now implement convolution. I decided to use conv2d through the function conv1d above since that was more intuitive - - self.context_emb_out = conv1d(input_= self.context_char_embs, output_size = self.FLAGS.char_out_size, width =self.FLAGS.window_width , stride=1, scope_name='char-cnn') - - self.context_emb_out = tf.nn.dropout(self.context_emb_out, self.keep_prob) - - print("Shape context embs after conv", self.context_emb_out.shape) - - self.context_emb_out = tf.reduce_sum(self.context_emb_out, axis = 1) - - self.context_emb_out = tf.reshape(self.context_emb_out, shape=(-1, self.FLAGS.context_len, self.FLAGS.char_out_size))# Desired shape is Batch_size, context_len, char_out_size - - print("Shape context embs after pooling", self.context_emb_out.shape) - - self.qn_emb_out = conv1d(input_=self.qn_char_embs, output_size=self.FLAGS.char_out_size, - width=self.FLAGS.window_width, stride=1, scope_name='char-cnn') #reuse weights b/w context and ques conv layers - - self.qn_emb_out = tf.nn.dropout(self.qn_emb_out, self.keep_prob) - - print("Shape qn embs after conv", self.qn_emb_out.shape) - - self.qn_emb_out = tf.reduce_sum(self.qn_emb_out, - axis=1) - - self.qn_emb_out = tf.reshape(self.qn_emb_out, shape=(-1, self.FLAGS.question_len, - self.FLAGS.char_out_size)) # Desired shape is Batch_size, question_len, char_out_size - - print("Shape qn embs after pooling", self.qn_emb_out.shape) - - return self.context_emb_out, self.qn_emb_out - - def build_graph(self): - """Builds the main part of the graph for the model, starting from the input embeddings to the final distributions for the answer span. - - Defines: - self.logits_start, self.logits_end: Both tensors shape (batch_size, context_len). - These are the logits (i.e. values that are fed into the softmax function) for the start and end distribution. - Important: these are -large in the pad locations. Necessary for when we feed into the cross entropy function. - self.probdist_start, self.probdist_end: Both shape (batch_size, context_len). Each row sums to 1. - These are the result of taking (masked) softmax of logits_start and logits_end. - """ - - # Use a RNN to get hidden states for the context and the question - # Note: here the RNNEncoder is shared (i.e. the weights are the same) - # between the context and the question. - - - ###################################################### CHAR EMBEDDING ####################################### - if self.FLAGS.do_char_embed: - - self.context_emb_out, self.qn_emb_out = self.add_char_embeddings() - self.context_embs = tf.concat((self.context_embs, self.context_emb_out), axis = 2) - print("Shape - concatenated context embs", self.context_embs.shape) - - self.qn_embs = tf.concat((self.qn_embs, self.qn_emb_out), axis=2) - print("Shape - concatenated qn embs", self.qn_embs.shape) - - ###################################################### HIGHWAY LAYER ####################################### - if self.FLAGS.add_highway_layer: - last_dim_concat = self.context_embs.get_shape().as_list()[-1] - for i in range(2): - #add two highway layers or repeat process twice - self.context_embs = self.highway(self.context_embs, last_dim_concat, scope_name='highway', carry_bias=-1.0) - #reuse variables for qn_embs - self.qn_embs = self.highway(self.qn_embs, last_dim_concat, scope_name='highway', carry_bias=-1.0) - - ###################################################### RNN ENCODER ####################################### - encoder = RNNEncoder(self.FLAGS.hidden_size_encoder, self.keep_prob) - context_hiddens = encoder.build_graph(self.context_embs, self.context_mask, scopename='RNNEncoder') # (batch_size, context_len, hidden_size*2) - question_hiddens = encoder.build_graph(self.qn_embs, self.qn_mask, scopename='RNNEncoder') # (batch_size, question_len, hidden_size*2) - - ###################################################### CNN ENCODER ####################################### - if self.FLAGS.cnn_encoder: - ## Use CNN to also generate encodings - cnn_encoder = CNNEncoder(self.FLAGS.filter_size_encoder, self.keep_prob) - context_cnn_hiddens = cnn_encoder.build_graph(self.context_embs, self.FLAGS.context_len, scope_name='context-encoder') # (batch_size, context_len, hidden_size*2) - print("Shape - Context Encoder output", context_cnn_hiddens.shape) - - ques_cnn_hiddens = cnn_encoder.build_graph(self.qn_embs, self.FLAGS.question_len, - scope_name='ques-encoder') # (batch_size, context_len, hidden_size*2) - print("Shape - Ques Encoder output", ques_cnn_hiddens.shape) - - ## concat these vectors - context_hiddens = tf.concat((context_hiddens, context_cnn_hiddens), axis = 2) # Just use these output for now. Ignore the RNN output - question_hiddens = tf.concat((question_hiddens, ques_cnn_hiddens), axis = 2) # Just use these output for now - print("Shape - Context Hiddens", context_hiddens.shape) - - ###################################################### RNET QUESTION CONTEXT ATTENTION and SELF ATTENTION ####################################### - if self.FLAGS.rnet_attention: ##perform Question Passage and Self Matching attention from R-Net - - rnet_layer = Attention_Match_RNN(self.keep_prob, self.FLAGS.hidden_size_encoder, self.FLAGS.hidden_size_qp_matching, self.FLAGS.hidden_size_sm_matching) - - # Implement better question_passage matching - v_P = rnet_layer.build_graph_qp_matching(context_hiddens, question_hiddens, self.qn_mask, self.context_mask, self.FLAGS.context_len, self.FLAGS.question_len) - - h_P = rnet_layer.build_graph_sm_matching(context_hiddens, question_hiddens, self.qn_mask, self.context_mask, - self.FLAGS.context_len, self.FLAGS.question_len, v_P) - - # Blended reps for R-Net - blended_reps = tf.concat([context_hiddens, v_P, h_P], axis=2) # (batch_size, context_len, hidden_size*6) - - ###################################################### BIDAF ATTENTION AND MODELING LAYER ####################################### - elif self.FLAGS.bidaf_attention: - - attn_layer = BiDAF(self.keep_prob, self.FLAGS.hidden_size_encoder * 2) - attn_output = attn_layer.build_graph(question_hiddens, self.qn_mask, context_hiddens, - self.context_mask) # attn_output is shape (batch_size, context_len, hidden_size_encoder*6) - blended_reps = tf.concat([context_hiddens, attn_output], axis=2) # (batch_size, context_len, hidden_size_encoder*8) - - ## add a modeling layer - modeling_layer = RNNEncoder(self.FLAGS.hidden_size_modeling, self.keep_prob) - attention_hidden = modeling_layer.build_graph(blended_reps, - self.context_mask, scopename='bidaf_modeling') # (batch_size, context_len, hidden_size*2) - - blended_reps = attention_hidden # for the final layer - - ###################################################### BASELINE DOT PRODUCT ATTENTION ####################################### - else: ## perform baseline dot product attention - - # Use context hidden states to attend to question hidden states - Basic Attention - - last_dim = context_hiddens.get_shape().as_list()[-1] - print("last dim", last_dim) - - attn_layer = BasicAttn(self.keep_prob, last_dim, - last_dim) - _, attn_output = attn_layer.build_graph(question_hiddens, self.qn_mask, - context_hiddens) # attn_output is shape (batch_size, context_len, hidden_size*2) - - - # Concat attn_output to context_hiddens to get blended_reps - blended_reps = tf.concat([context_hiddens, attn_output], axis=2) # (batch_size, context_len, hidden_size*4) - - ###################################################### RNET QUESTION POOLING and ANSWER POINTER ####################################### - if self.FLAGS.answer_pointer_RNET: ##Use Answer Pointer Module from R-Net - - if self.FLAGS.rnet_attention: - # different attention size for R-Net final layer - hidden_size_attn = 2 * self.FLAGS.hidden_size_encoder + self.FLAGS.hidden_size_qp_matching + 2 * self.FLAGS.hidden_size_sm_matching # combined size of blended reps - else: - hidden_size_attn = 4 * self.FLAGS.hidden_size_encoder # Final attention size for baseline model - - attn_ptr_layer = Answer_Pointer(self.keep_prob, self.FLAGS.hidden_size_encoder, - self.FLAGS.question_len, hidden_size_attn) - - p, logits = attn_ptr_layer.build_graph_answer_pointer(context_hiddens, question_hiddens, self.qn_mask, - self.context_mask, - self.FLAGS.context_len, self.FLAGS.question_len, - blended_reps) - - self.logits_start = logits[0] - self.probdist_start = p[0] - - self.logits_end = logits[1] - self.probdist_end = p[1] - - ###################################################### BASELINE FULLY CONNECTED ANSWER PREDICTION ####################################### - else: # Use baseline fully connected for answer start and end prediction - - # Apply fully connected layer to each blended representation - # Note, blended_reps_final corresponds to b' in the handout - # Note, tf.contrib.layers.fully_connected applies a ReLU non-linarity here by default - blended_reps_final = tf.contrib.layers.fully_connected(blended_reps, num_outputs=self.FLAGS.hidden_size_fully_connected) # blended_reps_final is shape (batch_size, context_len, hidden_size) - - # Use softmax layer to compute probability distribution for start location - # Note this produces self.logits_start and self.probdist_start, both of which have shape (batch_size, context_len) - with vs.variable_scope("StartDist"): - softmax_layer_start = SimpleSoftmaxLayer() - self.logits_start, self.probdist_start = softmax_layer_start.build_graph(blended_reps_final, self.context_mask) - - # Use softmax layer to compute probability distribution for end location - # Note this produces self.logits_end and self.probdist_end, both of which have shape (batch_size, context_len) - with vs.variable_scope("EndDist"): - softmax_layer_end = SimpleSoftmaxLayer() - self.logits_end, self.probdist_end = softmax_layer_end.build_graph(blended_reps_final, self.context_mask) - - - def add_loss(self): - """ - Add loss computation to the graph. - - Uses: - self.logits_start: shape (batch_size, context_len) - IMPORTANT: Assumes that self.logits_start is masked (i.e. has -large in masked locations). - That's because the tf.nn.sparse_softmax_cross_entropy_with_logits - function applies softmax and then computes cross-entropy loss. - So you need to apply masking to the logits (by subtracting large - number in the padding location) BEFORE you pass to the - sparse_softmax_cross_entropy_with_logits function. - - self.ans_span: shape (batch_size, 2) - Contains the gold start and end locations - - Defines: - self.loss_start, self.loss_end, self.loss: all scalar tensors - """ - with vs.variable_scope("loss"): - - # Calculate loss for prediction of start position - loss_start = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits_start, labels=self.ans_span[:, 0]) # loss_start has shape (batch_size) - self.loss_start = tf.reduce_mean(loss_start) # scalar. avg across batch - tf.summary.scalar('loss_start', self.loss_start) # log to tensorboard - - # Calculate loss for prediction of end position - loss_end = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits_end, labels=self.ans_span[:, 1]) - self.loss_end = tf.reduce_mean(loss_end) - tf.summary.scalar('loss_end', self.loss_end) - - # Add the two losses - self.loss = self.loss_start + self.loss_end - tf.summary.scalar('loss', self.loss) - - - def run_train_iter(self, session, batch, summary_writer): - """ - This performs a single training iteration (forward pass, loss computation, backprop, parameter update) - - Inputs: - session: TensorFlow session - batch: a Batch object - summary_writer: for Tensorboard - - Returns: - loss: The loss (averaged across the batch) for this batch. - global_step: The current number of training iterations we've done - param_norm: Global norm of the parameters - gradient_norm: Global norm of the gradients - """ - # Match up our input data with the placeholders - input_feed = {} - input_feed[self.context_ids] = batch.context_ids - input_feed[self.context_mask] = batch.context_mask - input_feed[self.qn_ids] = batch.qn_ids - input_feed[self.qn_mask] = batch.qn_mask - input_feed[self.ans_span] = batch.ans_span - input_feed[self.keep_prob] = 1.0 - self.FLAGS.dropout # apply dropout - if self.FLAGS.do_char_embed: - input_feed[self.char_ids_context] = self.padded_char_ids(batch, batch.context_ids) - input_feed[self.char_ids_qn] = self.padded_char_ids(batch, batch.qn_ids) - - # output_feed contains the things we want to fetch. - output_feed = [self.updates, self.summaries, self.loss, self.global_step, self.param_norm, self.gradient_norm] - - # Run the model - [_, summaries, loss, global_step, param_norm, gradient_norm] = session.run(output_feed, input_feed) - - # All summaries in the graph are added to Tensorboard - summary_writer.add_summary(summaries, global_step) - - return loss, global_step, param_norm, gradient_norm - - - def get_loss(self, session, batch): - """ - Run forward-pass only; get loss. - - Inputs: - session: TensorFlow session - batch: a Batch object - - Returns: - loss: The loss (averaged across the batch) for this batch - """ - - input_feed = {} - input_feed[self.context_ids] = batch.context_ids - input_feed[self.context_mask] = batch.context_mask - input_feed[self.qn_ids] = batch.qn_ids - input_feed[self.qn_mask] = batch.qn_mask - input_feed[self.ans_span] = batch.ans_span - if self.FLAGS.do_char_embed: - input_feed[self.char_ids_context] = self.padded_char_ids(batch, batch.context_ids) - input_feed[self.char_ids_qn] = self.padded_char_ids(batch, batch.qn_ids) - - # note you don't supply keep_prob here, so it will default to 1 i.e. no dropout - - output_feed = [self.loss] - - [loss] = session.run(output_feed, input_feed) - - return loss - - - def get_prob_dists(self, session, batch): - """ - Run forward-pass only; get probability distributions for start and end positions. - - Inputs: - session: TensorFlow session - batch: Batch object - - Returns: - probdist_start and probdist_end: both shape (batch_size, context_len) - """ - input_feed = {} - input_feed[self.context_ids] = batch.context_ids - input_feed[self.context_mask] = batch.context_mask - input_feed[self.qn_ids] = batch.qn_ids - input_feed[self.qn_mask] = batch.qn_mask - if self.FLAGS.do_char_embed: - input_feed[self.char_ids_context] = self.padded_char_ids(batch, batch.context_ids) - input_feed[self.char_ids_qn] = self.padded_char_ids(batch, batch.qn_ids) - - # note you don't supply keep_prob here, so it will default to 1 i.e. no dropout - - output_feed = [self.probdist_start, self.probdist_end] - [probdist_start, probdist_end] = session.run(output_feed, input_feed) - return probdist_start, probdist_end - - - def get_start_end_pos(self, session, batch): - """ - Run forward-pass only; get the most likely answer span. - - Inputs: - session: TensorFlow session - batch: Batch object - - Returns: - start_pos, end_pos: both numpy arrays shape (batch_size). - The most likely start and end positions for each example in the batch. - """ - # Get start_dist and end_dist, both shape (batch_size, context_len) - start_dist, end_dist = self.get_prob_dists(session, batch) - ###################################################### SMARTER SPAN SELECTION ####################################### - if self.FLAGS.smart_span: - - curr_batch_size = batch.batch_size - start_pos = np.empty(shape = (curr_batch_size), dtype=int) - end_pos = np.empty(shape=(curr_batch_size), dtype=int) - - for j in range(curr_batch_size): # for each row - ## Take argmax of start and end dist in a window such that i <= j <= i + 15 - maxprod = 0 - chosen_start = 0 - chosen_end = 0 - for i in range(self.FLAGS.context_len-16): - end_dist_subset = end_dist[j,i:i+16] - end_prob_max = np.amax(end_dist_subset) - end_idx = np.argmax(end_dist_subset) - start_prob = start_dist[j,i] - prod = end_prob_max*start_prob - - # print("Shape end, start:", end_prob_max.shape, start_prob.shape) - - if prod > maxprod: - maxprod = prod - chosen_start = i - chosen_end = chosen_start+end_idx - - start_pos[j] = chosen_start - # end_idx = np.argmax(end_dist[j:chosen_start:chosen_start+16]) - # print("Chosen end", chosen_start+end_idx) - end_pos[j] = chosen_end - - ## add sanity check - delta = end_pos[j] - start_pos[j] - if delta < 0 or delta > 16: - print("Error! Please look ") - - # print("Shape end, start matrix:", start_pos.shape, end_pos.shape) - # print("Start and end matrix:", start_pos, end_pos) - - else: - - # Take argmax to get start_pos and end_post, both shape (batch_size) - start_pos = np.argmax(start_dist, axis=1) - end_pos = np.argmax(end_dist, axis=1) - - return start_pos, end_pos - - ## Helper Functions for Char CNN - - def create_char_dicts(self, CHAR_PAD_ID=0, CHAR_UNK_ID = 1, _CHAR_PAD = '*', _CHAR_UNK = '$' ): - - unique_chars = ['!', '"', '#', '$', '%', '&', "'", '(', ')', '+', ',', '-', '.', '/', '0', '1', '2', '3', - '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '[', ']', '^', 'a', 'b', 'c', 'd', - 'e' , 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '~', ] # based on analysis in jupyter notebook - - num_chars = len(unique_chars) - - idx2char = dict(enumerate(unique_chars, 2)) ##reserve first 2 spots - idx2char[CHAR_PAD_ID] = _CHAR_PAD - idx2char[CHAR_UNK_ID] = _CHAR_UNK - - ##Create reverse char2idx - char2idx = {v: k for k, v in idx2char.iteritems()} - return char2idx, idx2char, num_chars - - def word_to_token_ids(self, word): - """Turns a word into char idxs - e.g. "know" -> [9, 32, 16, 96] - Note any token that isn't in the char2idx mapping gets mapped to the id for UNK_CHAR - """ - char2idx, idx2char, _ = self.create_char_dicts() - char_tokens = list(word) # list of chars in word - char_ids = [char2idx.get(w, 1) for w in char_tokens] - return char_tokens, char_ids - - - def padded_char_ids(self,batch, token_ids): # have to use token_ids since only those are padded - - charids_batch = [] - for i in range(batch.batch_size): - charids_line = [] - #for each example - token_row = token_ids[i,:] - # print("Each token row is", token_row) - # print("Shape token row is ", token_row.shape) - for j in range(len(token_row)): - id = token_row[j] - # print("each id is:" ,id) - word = self.id2word[id] # convert token id to word - _, char_ids = self.word_to_token_ids(word) - # for each word we get char_ids but they maybe different_length - if len(char_ids) < self.FLAGS.word_max_len: #pad with CHAR pad tokens - while len(char_ids) < self.FLAGS.word_max_len: - char_ids.append(0) - pad_char_ids = char_ids - - else: # if longer truncate to word max len - pad_char_ids = char_ids[:self.FLAGS.word_max_len] - - charids_line.append(pad_char_ids) - charids_batch.append(charids_line) - - return charids_batch - - def matrix_multiplication(self, mat, weight): - # [batch_size, seq_len, hidden_size] * [hidden_size, p] = [batch_size, seq_len, p] - - mat_shape = mat.get_shape().as_list() # shape - ijk - weight_shape = weight.get_shape().as_list() # shape -kl - assert (mat_shape[-1] == weight_shape[0]) - mat_reshape = tf.reshape(mat, [-1, mat_shape[-1]]) # [batch_size * n, m] - mul = tf.matmul(mat_reshape, weight) # [batch_size * n, p] - return tf.reshape(mul, [-1, mat_shape[1], weight_shape[-1]]) # reshape to batch_size, seq_len, p - - def highway(self, x, size, scope_name, carry_bias=-1.0): - - with tf.variable_scope(scope_name, reuse=tf.AUTO_REUSE): - W_T = tf.Variable(tf.truncated_normal([size, size], stddev=0.1), name="weight_transform") - b_T = tf.Variable(tf.constant(carry_bias, shape=[size]), name="bias_transform") - - W = tf.Variable(tf.truncated_normal([size, size], stddev=0.1), name="weight") - b = tf.Variable(tf.constant(0.1, shape=[size]), name="bias") - - T = tf.sigmoid(self.matrix_multiplication(x, W_T) + b_T, name="transform_gate") - H = tf.nn.relu(self.matrix_multiplication(x, W) + b, name="activation") - - print("shape H, T: ", H.shape, T.shape) - C = tf.subtract(1.0, T, name="carry_gate") - - y = tf.add(tf.multiply(H, T), tf.multiply(x, C), "y") - return y - - - def get_dev_loss(self, session, dev_context_path, dev_qn_path, dev_ans_path): - """ - Get loss for entire dev set. - - Inputs: - session: TensorFlow session - dev_qn_path, dev_context_path, dev_ans_path: paths to the dev.{context/question/answer} data files - - Outputs: - dev_loss: float. Average loss across the dev set. - """ - logging.info("Calculating dev loss...") - tic = time.time() - loss_per_batch, batch_lengths = [], [] - - # Iterate over dev set batches - # Note: here we set discard_long=True, meaning we discard any examples - # which are longer than our context_len or question_len. - # We need to do this because if, for example, the true answer is cut - # off the context, then the loss function is undefined. - for batch in get_batch_generator(self.word2id, dev_context_path, dev_qn_path, dev_ans_path, self.FLAGS.batch_size, context_len=self.FLAGS.context_len, question_len=self.FLAGS.question_len, discard_long=True): - - # Get loss for this batch - loss = self.get_loss(session, batch) - curr_batch_size = batch.batch_size - loss_per_batch.append(loss * curr_batch_size) - batch_lengths.append(curr_batch_size) - - # Calculate average loss - total_num_examples = sum(batch_lengths) - toc = time.time() - print "Computed dev loss over %i examples in %.2f seconds" % (total_num_examples, toc-tic) - - # Overall loss is total loss divided by total number of examples - dev_loss = sum(loss_per_batch) / float(total_num_examples) - - return dev_loss - - - def check_f1_em(self, session, context_path, qn_path, ans_path, dataset, num_samples=100, print_to_screen=False): - """ - Sample from the provided (train/dev) set. - For each sample, calculate F1 and EM score. - Return average F1 and EM score for all samples. - Optionally pretty-print examples. - - Note: This function is not quite the same as the F1/EM numbers you get from "official_eval" mode. - This function uses the pre-processed version of the e.g. dev set for speed, - whereas "official_eval" mode uses the original JSON. Therefore: - 1. official_eval takes your max F1/EM score w.r.t. the three reference answers, - whereas this function compares to just the first answer (which is what's saved in the preprocessed data) - 2. Our preprocessed version of the dev set is missing some examples - due to tokenization issues (see squad_preprocess.py). - "official_eval" includes all examples. - - Inputs: - session: TensorFlow session - qn_path, context_path, ans_path: paths to {dev/train}.{question/context/answer} data files. - dataset: string. Either "train" or "dev". Just for logging purposes. - num_samples: int. How many samples to use. If num_samples=0 then do whole dataset. - print_to_screen: if True, pretty-prints each example to screen - - Returns: - F1 and EM: Scalars. The average across the sampled examples. - """ - logging.info("Calculating F1/EM for %s examples in %s set..." % (str(num_samples) if num_samples != 0 else "all", dataset)) - - f1_total = 0. - em_total = 0. - example_num = 0 - - tic = time.time() - - # Note here we select discard_long=False because we want to sample from the entire dataset - # That means we're truncating, rather than discarding, examples with too-long context or questions - for batch in get_batch_generator(self.word2id, context_path, qn_path, ans_path, self.FLAGS.batch_size, context_len=self.FLAGS.context_len, question_len=self.FLAGS.question_len, discard_long=False): - - pred_start_pos, pred_end_pos = self.get_start_end_pos(session, batch) - - # Convert the start and end positions to lists length batch_size - pred_start_pos = pred_start_pos.tolist() # list length batch_size - pred_end_pos = pred_end_pos.tolist() # list length batch_size - - for ex_idx, (pred_ans_start, pred_ans_end, true_ans_tokens) in enumerate(zip(pred_start_pos, pred_end_pos, batch.ans_tokens)): - example_num += 1 - - # Get the predicted answer - # Important: batch.context_tokens contains the original words (no UNKs) - # You need to use the original no-UNK version when measuring F1/EM - pred_ans_tokens = batch.context_tokens[ex_idx][pred_ans_start : pred_ans_end + 1] - pred_answer = " ".join(pred_ans_tokens) - - # Get true answer (no UNKs) - true_answer = " ".join(true_ans_tokens) - - # Calc F1/EM - f1 = f1_score(pred_answer, true_answer) - em = exact_match_score(pred_answer, true_answer) - f1_total += f1 - em_total += em - - # Optionally pretty-print - if print_to_screen: - print_example(self.word2id, batch.context_tokens[ex_idx], batch.qn_tokens[ex_idx], batch.ans_span[ex_idx, 0], batch.ans_span[ex_idx, 1], pred_ans_start, pred_ans_end, true_answer, pred_answer, f1, em) - - if num_samples != 0 and example_num >= num_samples: - break - - if num_samples != 0 and example_num >= num_samples: - break - - f1_total /= example_num - em_total /= example_num - - toc = time.time() - logging.info("Calculating F1/EM for %i examples in %s set took %.2f seconds" % (example_num, dataset, toc-tic)) - - return f1_total, em_total - - - def train(self, session, train_context_path, train_qn_path, train_ans_path, dev_qn_path, dev_context_path, dev_ans_path): - """ - Main training loop. - - Inputs: - session: TensorFlow session - {train/dev}_{qn/context/ans}_path: paths to {train/dev}.{context/question/answer} data files - """ - - # Print number of model parameters - tic = time.time() - params = tf.trainable_variables() - num_params = sum(map(lambda t: np.prod(tf.shape(t.value()).eval()), params)) - toc = time.time() - logging.info("Number of params: %d (retrieval took %f secs)" % (num_params, toc - tic)) - - # We will keep track of exponentially-smoothed loss - exp_loss = None - - # Checkpoint management. - # We keep one latest checkpoint, and one best checkpoint (early stopping) - checkpoint_path = os.path.join(self.FLAGS.train_dir, "qa.ckpt") - bestmodel_dir = os.path.join(self.FLAGS.train_dir, "best_checkpoint") - bestmodel_ckpt_path = os.path.join(bestmodel_dir, "qa_best.ckpt") - best_dev_f1 = None - best_dev_em = None - - # for TensorBoard - summary_writer = tf.summary.FileWriter(self.FLAGS.train_dir, session.graph) - - epoch = 0 - - logging.info("Beginning training loop...") - while self.FLAGS.num_epochs == 0 or epoch < self.FLAGS.num_epochs: - epoch += 1 - epoch_tic = time.time() - - # Loop over batches - for batch in get_batch_generator(self.word2id, train_context_path, train_qn_path, train_ans_path, self.FLAGS.batch_size, context_len=self.FLAGS.context_len, question_len=self.FLAGS.question_len, discard_long=True): - - # Run training iteration - iter_tic = time.time() - loss, global_step, param_norm, grad_norm = self.run_train_iter(session, batch, summary_writer) - iter_toc = time.time() - iter_time = iter_toc - iter_tic - - # Update exponentially-smoothed loss - if not exp_loss: # first iter - exp_loss = loss - else: - exp_loss = 0.99 * exp_loss + 0.01 * loss - - # Sometimes print info to screen - if global_step % self.FLAGS.print_every == 0: - logging.info( - 'epoch %d, iter %d, loss %.5f, smoothed loss %.5f, grad norm %.5f, param norm %.5f, batch time %.3f' % - (epoch, global_step, loss, exp_loss, grad_norm, param_norm, iter_time)) - - # Sometimes save model - if global_step % self.FLAGS.save_every == 0: - logging.info("Saving to %s..." % checkpoint_path) - self.saver.save(session, checkpoint_path, global_step=global_step) - - # Sometimes evaluate model on dev loss, train F1/EM and dev F1/EM - if global_step % self.FLAGS.eval_every == 0: - - # Get loss for entire dev set and log to tensorboard - dev_loss = self.get_dev_loss(session, dev_context_path, dev_qn_path, dev_ans_path) - logging.info("Epoch %d, Iter %d, dev loss: %f" % (epoch, global_step, dev_loss)) - write_summary(dev_loss, "dev/loss", summary_writer, global_step) - - - # Get F1/EM on train set and log to tensorboard - train_f1, train_em = self.check_f1_em(session, train_context_path, train_qn_path, train_ans_path, "train", num_samples=1000) - logging.info("Epoch %d, Iter %d, Train F1 score: %f, Train EM score: %f" % (epoch, global_step, train_f1, train_em)) - write_summary(train_f1, "train/F1", summary_writer, global_step) - write_summary(train_em, "train/EM", summary_writer, global_step) - - - # Get F1/EM on dev set and log to tensorboard - dev_f1, dev_em = self.check_f1_em(session, dev_context_path, dev_qn_path, dev_ans_path, "dev", num_samples=0) - logging.info("Epoch %d, Iter %d, Dev F1 score: %f, Dev EM score: %f" % (epoch, global_step, dev_f1, dev_em)) - write_summary(dev_f1, "dev/F1", summary_writer, global_step) - write_summary(dev_em, "dev/EM", summary_writer, global_step) - - - # Early stopping based on dev EM. You could switch this to use F1 instead. - if best_dev_em is None or dev_em > best_dev_em: - best_dev_em = dev_em - logging.info("Saving to %s..." % bestmodel_ckpt_path) - self.bestmodel_saver.save(session, bestmodel_ckpt_path, global_step=global_step) - - - epoch_toc = time.time() - logging.info("End of epoch %i. Time for epoch: %f" % (epoch, epoch_toc-epoch_tic)) - - sys.stdout.flush() - - - -def write_summary(value, tag, summary_writer, global_step): - """Write a single summary value to tensorboard""" - summary = tf.Summary() - summary.value.add(tag=tag, simple_value=value) - summary_writer.add_summary(summary, global_step) diff --git a/codelab_details b/codelab_details deleted file mode 100644 index ea3e10d..0000000 --- a/codelab_details +++ /dev/null @@ -1,41 +0,0 @@ -codelab -username: pdwivedi -pwd: sneha123 -cs224n-priya_alex - -codelab submission handout -http://web.stanford.edu/class/cs224n/default_project/codalab_instructions.pdf - -python code/main.py --experiment_name=self_attention --mode=official_eval \ ---json_in_path=data/tiny-dev.json \ ---ckpt_load_dir=experiments/self_attention/best_checkpoint - -python code/evaluate.py data/tiny-dev.json predictions.json - -cl run --name gen-answers --request-docker-image abisee/cs224n-dfp:v4 \ -:code :best_checkpoint glove.txt:0x97c870/glove.6B.100d.txt data.json:0x4870af \ -'python code/main.py --experiment_name=rnet_sm --mode=official_eval \ ---glove_path=glove.txt --json_in_path=data.json --ckpt_load_dir=best_checkpoint' - -# Look at the status of the run -cl info --verbose gen-answers - -# Blocks until the job is complete, while tailing the output -cl wait --tail gen-answers - -# Inspect the resulting files -cl cat gen-answers # List the files -cl cat gen-answers/stderr # Inspect stderr -cl cat gen-answers/stdout # Inspect stdout -cl cat gen-answers/predictions.json # Inspect specific file - -#Submit to dev leaderboard -cl edit gen-answers -T cs224n-win18-dev --description "better model 1" - -#Dev leaderboard -http://cs224n-win18-leaderboard.westus.cloudapp.azure.com/dev.html - - -cl run --name run-eval --request-docker-image abisee/cs224n-dfp:v4 \ -:code data.json:0x4870af preds.json:gen-answers/predictions.json \ -'python code/evaluate.py data.json preds.json' diff --git a/codelab_instructions b/codelab_instructions deleted file mode 100644 index 83d4e9d..0000000 --- a/codelab_instructions +++ /dev/null @@ -1,8 +0,0 @@ - -Evaluate Locally on Tiny Json - -python code/main.py --experiment_name=rnet_sm --mode=official_eval \ ---json_in_path=data/tiny-dev.json \ ---ckpt_load_dir=experiments/rnet_sm/best_checkpoint - -python code/evaluate.py data/tiny-dev.json predictions.json diff --git a/create_small_sets.ipynb b/create_small_sets.ipynb deleted file mode 100644 index d401289..0000000 --- a/create_small_sets.ipynb +++ /dev/null @@ -1,219 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "from __future__ import division\n", - "import os\n", - "import glob\n", - "import numpy as np\n", - "from random import randint\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Break training set" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "86326\n" - ] - } - ], - "source": [ - "train_context = 'data/train.context'\n", - "\n", - "train_context_list = []\n", - "\n", - "with open(train_context) as fp: \n", - " lines = fp.readlines()\n", - "\n", - "for line in lines:\n", - " train_context_list.append(line)\n", - "\n", - "print(len(train_context_list))" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "86326\n" - ] - } - ], - "source": [ - "train_question = 'data/train.question'\n", - "\n", - "train_question_list = []\n", - "\n", - "with open(train_question) as fp: \n", - " lines = fp.readlines()\n", - "\n", - "for line in lines:\n", - " train_question_list.append(line)\n", - "\n", - "print(len(train_question_list))" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "86326\n" - ] - } - ], - "source": [ - "train_span = 'data/train.span'\n", - "train_span_list = []\n", - "\n", - "with open(train_span) as fp: \n", - " lines = fp.readlines()\n", - "\n", - "for line in lines:\n", - " train_span_list.append(line)\n", - "\n", - "print(len(train_span_list))" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "4662\n", - "(20000, 20000, 20000)\n" - ] - } - ], - "source": [ - "new_size_train = 20000\n", - "\n", - "rand = randint(0,86326-new_size_train)\n", - "print(rand)\n", - "\n", - "\n", - "train_context_subset = train_context_list[rand:rand+new_size_train]\n", - "train_question_subset = train_question_list[rand:rand+new_size_train]\n", - "train_span_subset = train_span_list[rand:rand+new_size_train]\n", - "print(len(train_context_subset), len(train_question_subset), len(train_span_subset))" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['other researchers point out that finding a difference in disease prevalence between two socially defined groups does not necessarily imply genetic causation of the difference . they suggest that medical practices should maintain their focus on the individual rather than an individual \\'s membership to any group . they argue that overemphasizing genetic contributions to health disparities carries various risks such as reinforcing stereotypes , promoting racism or ignoring the contribution of non-genetic factors to health disparities . international epidemiological data show that living conditions rather than race make the biggest difference in health outcomes even for diseases that have \" race-specific \" treatments . some studies have found that patients are reluctant to accept racial categorization in medical practice .\\n', 'chivalry and the ethos of courtly love developed in royal and noble courts . this culture was expressed in the vernacular languages rather than latin , and comprised poems , stories , legends , and popular songs spread by troubadours , or wandering minstrels . often the stories were written down in the chansons de geste , or \" songs of great deeds \" , such as the song of roland or the song of hildebrand . secular and religious histories were also produced . geoffrey of monmouth ( d. c. 1155 ) composed his historia regum britanniae , a collection of stories and legends about arthur . other works were more clearly history , such as otto von freising \\'s ( d. 1158 ) gesta friderici imperatoris detailing the deeds of emperor frederick barbarossa , or william of malmesbury \\'s ( d. c. 1143 ) gesta regum on the kings of england .\\n', \"at the time of the roman empire , about two thousand years ago , various tribes , which spoke celtic dialects of the insular celtic group , were inhabiting the islands . the romans expanded their civilisation to control southern great britain but were impeded in advancing any further , building hadrian 's wall to mark the northern frontier of their empire in 122 ad . at that time , ireland was populated by a people known as hiberni , the northern third or so of great britain by a people known as picts and the southern two thirds by britons .\\n\"]\n" - ] - } - ], - "source": [ - "print(train_context_subset[:3])" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "thefile = open('data/train_subset.context', 'w')\n", - "\n", - "for item in train_context_subset:\n", - " thefile.write(\"%s\" % item)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "thefile = open('data/train_subset.question', 'w')\n", - "\n", - "for item in train_question_subset:\n", - " thefile.write(\"%s\" % item)" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "thefile = open('data/train_subset.span', 'w')\n", - "\n", - "for item in train_span_subset:\n", - " thefile.write(\"%s\" % item)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Repeat the steps on a dev subset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/run_instructions b/run_instructions deleted file mode 100644 index 111cac8..0000000 --- a/run_instructions +++ /dev/null @@ -1,127 +0,0 @@ -RNET - -python main.py --mode=train --experiment_name=rnet_span --dropout=0.20 --batch_size=20 --hidden_size_encoder=200 --embedding_size=300 --do_char_embed=False --add_highway_layer=False --rnet_attention=True --bidaf_attention=False --answer_pointer_RNET=True --smart_span=True - -RNET - eval model - -python code/main.py --experiment_name=rnet_span --dropout=0.20 --batch_size=20 --hidden_size_encoder=200 --embedding_size=300 --do_char_embed=False --add_highway_layer=False --rnet_attention=True --bidaf_attention=False --answer_pointer_RNET=True --smart_span=True--mode=official_eval \ ---json_in_path=data/tiny-dev.json \ ---json_out_path=predictions_rnet.json \ ---ckpt_load_dir=experiments/rnet_best/best_checkpoint - - -Solo submission using main.py for RNET-SPAN -python code/main.py --experiment_name=rnet_span --dropout=0.20 --batch_size=20 --hidden_size_encoder=200 --embedding_size=300 --do_char_embed=False --add_highway_layer=False --rnet_attention=True --bidaf_attention=False --answer_pointer_RNET=True --smart_span=True --mode=official_eval \ ---json_in_path=data/tiny-dev.json \ ---json_out_path=predictions.json \ ---ckpt_load_dir=experiments/rnet_span/best_checkpoint - -on codalab - -cl run --name gen-answers-rnet --request-docker-image abisee/cs224n-dfp:v4 \ -:code :best_checkpoint glove.txt:0x97c870/glove.6B.300d.txt data.json:0x4870af \ -'python code/main.py --experiment_name=rnet_span --dropout=0.20 --batch_size=20 --hidden_size_encoder=200 --embedding_size=300 --do_char_embed=False --add_highway_layer=False --rnet_attention=True --bidaf_attention=False --answer_pointer_RNET=True --smart_span=True --mode=official_eval \ ---glove_path=glove.txt --json_in_path=data.json --ckpt_load_dir=best_checkpoint' --request-cpus 1 --request-memory 4g --request-disk 1g --request-time 1d - -cl info --verbose gen-answers-rnet - -cl wait --tail gen-answers-rnet - - -cl cat gen-answers-rnet -cl cat gen-answers-rnet/stderr -cl cat gen-answers-rnet/stdout -cl cat gen-answers-rnet/predictions.json - - - -### For BIDAF best - -python code/main.py --experiment_name=bidaf_best --dropout=0.15 --batch_size=60 --hidden_size_encoder=150 --embedding_size=100 --do_char_embed=False --add_highway_layer=True --rnet_attention=False --bidaf_attention=True --answer_pointer_RNET=False --smart_span=True --hidden_size_modeling=150 --mode=train - -BIDAF - eval model on Local - -python code/main.py --experiment_name=bidaf_best --dropout=0.15 --batch_size=60 --hidden_size_encoder=150 --embedding_size=100 --do_char_embed=False --add_highway_layer=True --rnet_attention=False --bidaf_attention=True --answer_pointer_RNET=False --smart_span=True --hidden_size_modeling=150 --mode=official_eval \ ---json_in_path=data/tiny-dev.json \ ---json_out_path=predictions_bidaf.json \ ---ckpt_load_dir=experiments/bidaf_best/best_checkpoint - - -python code/evaluate.py data/tiny-dev.json predictions_bidaf.json - -on codalab - -cl run --name gen-answers-bidaf --request-docker-image abisee/cs224n-dfp:v4 \ -code:code best_checkpoint:best_checkpoint glove.txt:0x97c870/glove.6B.100d.txt data.json:0x4870af \ -'python code/main.py --experiment_name=bidaf_best --dropout=0.15 --batch_size=60 --hidden_size_encoder=150 --embedding_size=100 --do_char_embed=False --add_highway_layer=True --rnet_attention=False --bidaf_attention=True --answer_pointer_RNET=False --smart_span=True --hidden_size_modeling=150 --mode=official_eval \ ---glove_path=glove.txt --json_in_path=data.json --ckpt_load_dir=best_checkpoint' --request-cpus 1 --request-memory 10g --request-disk 1g --request-time 1d - -cl info --verbose gen-answers-bidaf - -cl wait --tail gen-answers-bidaf - -cl cat gen-answers-bidaf -cl cat gen-answers-bidaf/stderr -cl cat gen-answers-bidaf/stdout -cl cat gen-answers-bidaf/predictions.json - -Post to dev leaderboard - -cl edit gen-answers-bidaf -T cs224n-win18-dev --description "best model 1" - -cl edit gen-answers-bidaf -T cs224n-win18-test --description "best model 1" - - - -Ensemble model - -cl run --name gen-answers-bidaf-prob --request-docker-image abisee/cs224n-dfp:v4 \ -code:code best_checkpoint:best_checkpoint glove.txt:0x97c870/glove.6B.100d.txt data.json:0x4870af \ -'python code/main_ensemble.py --experiment_name=bidaf_best --dropout=0.15 --batch_size=60 --hidden_size_encoder=150 --embedding_size=100 --do_char_embed=False --add_highway_layer=True --rnet_attention=False --bidaf_attention=True --answer_pointer_RNET=False --smart_span=True --hidden_size_modeling=150 --mode=official_eval \ ---glove_path=glove.txt --json_in_path=data.json --ckpt_load_dir=best_checkpoint' --request-cpus 1 --request-memory 10g --request-disk 1g --request-time 1d - - -cl info --verbose gen-answers-bidaf-prob - -cl wait --tail gen-answers-bidaf-prob - -cl cat gen-answers-bidaf-prob/predictions.json - -### RNET ### -cl upload experiments/rnet_span/best_checkpoint - - -cl run --name gen-answers-rnet-prob --request-docker-image abisee/cs224n-dfp:v4 \ -:code :best_checkpoint glove.txt:0x97c870/glove.6B.300d.txt data.json:0x4870af \ -'python code/main_ensemble.py --experiment_name=rnet_span --dropout=0.20 --batch_size=20 --hidden_size_encoder=200 --embedding_size=300 --do_char_embed=False --add_highway_layer=False --rnet_attention=True --bidaf_attention=False --answer_pointer_RNET=True --smart_span=True --mode=official_eval \ ---glove_path=glove.txt --json_in_path=data.json --ckpt_load_dir=best_checkpoint' --request-cpus 1 --request-memory 15g --request-disk 1g --request-time 1d - -cl info --verbose gen-answers-rnet-prob - -cl wait --tail gen-answers-rnet-prob - -cl cat gen-answers-rnet-prob/predictions.json - - -### ENSEMBLE on Codalab###### - -cl run --name gen-answers-ensemble --request-docker-image abisee/cs224n-dfp:v4 \ -:code file1:gen-answers-bidaf-prob/predictions.json file2:gen-answers-rnet-prob/predictions.json \ -'python code/ensemble.py file1 file2' --request-cpus 1 --request-memory 10g --request-disk 1g --request-time 1d - -cl info --verbose gen-answers-ensemble - -cl cat gen-answers-ensemble/predictions.json - - -## Send ensemble to dev leaderboard -cl edit gen-answers-ensemble -T cs224n-win18-dev --description "ensemble 1" - - -## send to sanity check leaderboard - -cl edit gen-answers-ensemble -T cs224n-win18-sanity-check --description "best ensemble 1" -cl edit gen-answers-bidaf -T cs224n-win18-sanity-check --description "best model 1" - -cl edit gen-answers-ensemble -T cs224n-win18-test --description "ensemble sub 1" -