1

I am trying to scrape data of a match played between United and Sheffield United yesterday night in the premier league from understat.com. My goal is to fetch "shots per game". If you see understat.com, it has a match id for all the matches and I am using that match id to scrape the data using BS4 and requests. I have successfully located the class and got the raw data that I need to fetch in JSON format but it's giving me an error like "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)". Below is my code:

#Import packages and modules
import requests
from bs4 import BeautifulSoup
import json
import pandas as pd

#Set up the url and class to scrape
#scrape the single game shot
base_url = "https://understat.com/match/"

match_id = str(input('Please enter the match id: '))
url = base_url + match_id

print(url)

res = requests.get(url)
soup = BeautifulSoup(res.content, 'lxml')
scripts = soup.find_all('script')

print(scripts)

#only the shot data
strings = scripts[1].string
print(strings)

#strip symbols so we only have the json data
index_start = strings.index("('")+1
index_end = strings.index("')")

json_data = strings[index_start:index_end]
json_data = json_data.encode('utf8').decode('unicode_escape')

#convert string to json format
data = json.loads(json_data)
print(data)

Shots strings that I am trying to fetch: var shotsData = JSON.parse('\x7B\x22h\x22\x3A\x5B\x7B\x22id\x22\x3A\x22401340\x22,\x22minute\x22\x3A\x2210\x22,\x22result\x22\x3A\x22MissedShots\x22,\x22X\x22\x3A\x220.8390000152587891\x22,\x22Y\x22\x3A\x220.5379999923706055\x22,\x22xG\x22\x3A\x220.08287161588668823\x22,\x22player\x22\x3A\x22Marcus\x20Rashford\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x22556\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22RightFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22Aaron\x20Wan\x2DBissaka\x22,\x22lastAction\x22\x3A\x22Pass\x22\x7D,\x7B\x22id\x22\x3A\x22401342\x22,\x22minute\x22\x3A\x2216\x22,\x22result\x22\x3A\x22BlockedShot\x22,\x22X\x22\x3A\x220.9230000305175782\x22,\x22Y\x22\x3A\x220.705999984741211\x22,\x22xG\x22\x3A\x220.04452449828386307\x22,\x22player\x22\x3A\x22Mason\x20Greenwood\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x227490\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22LeftFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22Bruno\x20Fernandes\x22,\x22lastAction\x22\x3A\x22Chipped\x22\x7D,\x7B\x22id\x22\x3A\x22401344\x22,\x22minute\x22\x3A\x2226\x22,\x22result\x22\x3A\x22SavedShot\x22,\x22X\x22\x3A\x220.74\x22,\x22Y\x22\x3A\x220.37900001525878907\x22,\x22xG\x22\x3A\x220.019290726631879807\x22,\x22player\x22\x3A\x22Mason\x20Greenwood\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x227490\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22LeftFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22Aaron\x20Wan\x2DBissaka\x22,\x22lastAction\x22\x3A\x22Pass\x22\x7D,\x7B\x22id\x22\x3A\x22401345\x22,\x22minute\x22\x3A\x2227\x22,\x22result\x22\x3A\x22BlockedShot\x22,\x22X\x22\x3A\x220.8930000305175781\x22,\x22Y\x22\x3A\x220.34900001525878904\x22,\x22xG\x22\x3A\x220.07055725157260895\x22,\x22player\x22\x3A\x22Mason\x20Greenwood\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x227490\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22RightFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22Aaron\x20Wan\x2DBissaka\x22,\x22lastAction\x22\x3A\x22Pass\x22\x7D,\x7B\x22id\x22\x3A\x22401346\x22,\x22minute\x22\x3A\x2229\x22,\x22result\x22\x3A\x22MissedShots\x22,\x22X\x22\x3A\x220.919000015258789\x22,\x22Y\x22\x3A\x220.46599998474121096\x22,\x22xG\x22\x3A\x220.04169069603085518\x22,\x22player\x22\x3A\x22Harry\x20Maguire\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x221687\x22,\x22situation\x22\x3A\x22FromCorner\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22Head\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3Anull,\x22lastAction\x22\x3A\x22Foul\x22\x7D,\x7B\x22id\x22\x3A\x22401347\x22,\x22minute\x22\x3A\x2238\x22,\x22result\x22\x3A\x22BlockedShot\x22,\x22X\x22\x3A\x220.7169999694824218\x22,\x22Y\x22\x3A\x220.4370000076293945\x22,\x22xG\x22\x3A\x220.016049593687057495\x22,\x22player\x22\x3A\x22Bruno\x20Fernandes\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x221228\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22RightFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22Paul\x20Pogba\x22,\x22lastAction\x22\x3A\x22Pass\x22\x7D,\x7B\x22id\x22\x3A\x22401348\x22,\x22minute\x22\x3A\x2240\x22,\x22result\x22\x3A\x22SavedShot\x22,\x22X\x22\x3A\x220.9330000305175781\x22,\x22Y\x22\x3A\x220.485\x22,\x22xG\x22\x3A\x220.0721256285905838\x22,\x22player\x22\x3A\x22Anthony\x20Martial\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x22553\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22Head\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22Bruno\x20Fernandes\x22,\x22lastAction\x22\x3A\x22Pass\x22\x7D,\x7B\x22id\x22\x3A\x22401349\x22,\x22minute\x22\x3A\x2248\x22,\x22result\x22\x3A\x22MissedShots\x22,\x22X\x22\x3A\x220.925\x22,\x22Y\x22\x3A\x220.35700000762939454\x22,\x22xG\x22\x3A\x220.27813461422920227\x22,\x22player\x22\x3A\x22Mason\x20Greenwood\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x227490\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22RightFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22Bruno\x20Fernandes\x22,\x22lastAction\x22\x3A\x22Pass\x22\x7D,\x7B\x22id\x22\x3A\x22401350\x22,\x22minute\x22\x3A\x2251\x22,\x22result\x22\x3A\x22BlockedShot\x22,\x22X\x22\x3A\x220.769000015258789\x22,\x22Y\x22\x3A\x220.35200000762939454\x22,\x22xG\x22\x3A\x220.019760465249419212\x22,\x22player\x22\x3A\x22Aaron\x20Wan\x2DBissaka\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x225584\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22RightFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3Anull,\x22lastAction\x22\x3A\x22None\x22\x7D,\x7B\x22id\x22\x3A\x22401352\x22,\x22minute\x22\x3A\x2261\x22,\x22result\x22\x3A\x22SavedShot\x22,\x22X\x22\x3A\x220.6830000305175781\x22,\x22Y\x22\x3A\x220.5070000076293946\x22,\x22xG\x22\x3A\x220.032827332615852356\x22,\x22player\x22\x3A\x22Marcus\x20Rashford\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x22556\x22,\x22situation\x22\x3A\x22DirectFreekick\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22RightFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3Anull,\x22lastAction\x22\x3A\x22Standard\x22\x7D,\x7B\x22id\x22\x3A\x22401353\x22,\x22minute\x22\x3A\x2263\x22,\x22result\x22\x3A\x22BlockedShot\x22,\x22X\x22\x3A\x220.77\x22,\x22Y\x22\x3A\x220.6519999694824219\x22,\x22xG\x22\x3A\x220.061657458543777466\x22,\x22player\x22\x3A\x22Bruno\x20Fernandes\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x221228\x22,\x22situation\x22\x3A\x22DirectFreekick\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22RightFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3Anull,\x22lastAction\x22\x3A\x22Standard\x22\x7D,\x7B\x22id\x22\x3A\x22401354\x22,\x22minute\x22\x3A\x2263\x22,\x22result\x22\x3A\x22Goal\x22,\x22X\x22\x3A\x220.9180000305175782\x22,\x22Y\x22\x3A\x220.51\x22,\x22xG\x22\x3A\x220.04361902177333832\x22,\x22player\x22\x3A\x22Harry\x20Maguire\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x221687\x22,\x22situation\x22\x3A\x22FromCorner\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22Head\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22Alex\x20Telles\x22,\x22lastAction\x22\x3A\x22Aerial\x22\x7D,\x7B\x22id\x22\x3A\x22401357\x22,\x22minute\x22\x3A\x2276\x22,\x22result\x22\x3A\x22BlockedShot\x22,\x22X\x22\x3A\x220.7609999847412109\x22,\x22Y\x22\x3A\x220.6780000305175782\x22,\x22xG\x22\x3A\x220.016254646703600883\x22,\x22player\x22\x3A\x22Alex\x20Telles\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x221828\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22LeftFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3Anull,\x22lastAction\x22\x3A\x22None\x22\x7D,\x7B\x22id\x22\x3A\x22401358\x22,\x22minute\x22\x3A\x2276\x22,\x22result\x22\x3A\x22MissedShots\x22,\x22X\x22\x3A\x220.924000015258789\x22,\x22Y\x22\x3A\x220.5070000076293946\x22,\x22xG\x22\x3A\x220.10926949232816696\x22,\x22player\x22\x3A\x22Paul\x20Pogba\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x221740\x22,\x22situation\x22\x3A\x22FromCorner\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22Head\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22Alex\x20Telles\x22,\x22lastAction\x22\x3A\x22Cross\x22\x7D,\x7B\x22id\x22\x3A\x22401359\x22,\x22minute\x22\x3A\x2281\x22,\x22result\x22\x3A\x22MissedShots\x22,\x22X\x22\x3A\x220.8630000305175781\x22,\x22Y\x22\x3A\x220.5570000076293945\x22,\x22xG\x22\x3A\x220.06367684155702591\x22,\x22player\x22\x3A\x22Edinson\x20Cavani\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x223294\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22LeftFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3Anull,\x22lastAction\x22\x3A\x22None\x22\x7D,\x7B\x22id\x22\x3A\x22401360\x22,\x22minute\x22\x3A\x2285\x22,\x22result\x22\x3A\x22BlockedShot\x22,\x22X\x22\x3A\x220.86\x22,\x22Y\x22\x3A\x220.775\x22,\x22xG\x22\x3A\x220.02079056017100811\x22,\x22player\x22\x3A\x22Bruno\x20Fernandes\x22,\x22h_a\x22\x3A\x22h\x22,\x22player_id\x22\x3A\x221228\x22,\x22situation\x22\x3A\x22FromCorner\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22RightFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22Luke\x20Shaw\x22,\x22lastAction\x22\x3A\x22Pass\x22\x7D\x5D,\x22a\x22\x3A\x5B\x7B\x22id\x22\x3A\x22401341\x22,\x22minute\x22\x3A\x2215\x22,\x22result\x22\x3A\x22SavedShot\x22,\x22X\x22\x3A\x220.8690000152587891\x22,\x22Y\x22\x3A\x220.5479999923706055\x22,\x22xG\x22\x3A\x220.12690183520317078\x22,\x22player\x22\x3A\x22Billy\x20Sharp\x22,\x22h_a\x22\x3A\x22a\x22,\x22player_id\x22\x3A\x227712\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22LeftFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22John\x20Fleck\x22,\x22lastAction\x22\x3A\x22Throughball\x22\x7D,\x7B\x22id\x22\x3A\x22401343\x22,\x22minute\x22\x3A\x2222\x22,\x22result\x22\x3A\x22Goal\x22,\x22X\x22\x3A\x220.98\x22,\x22Y\x22\x3A\x220.49400001525878906\x22,\x22xG\x22\x3A\x220.15722613036632538\x22,\x22player\x22\x3A\x22Kean\x20Bryan\x22,\x22h_a\x22\x3A\x22a\x22,\x22player_id\x22\x3A\x229163\x22,\x22situation\x22\x3A\x22FromCorner\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22Head\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22John\x20Fleck\x22,\x22lastAction\x22\x3A\x22Cross\x22\x7D,\x7B\x22id\x22\x3A\x22401351\x22,\x22minute\x22\x3A\x2255\x22,\x22result\x22\x3A\x22MissedShots\x22,\x22X\x22\x3A\x220.8430000305175781\x22,\x22Y\x22\x3A\x220.29600000381469727\x22,\x22xG\x22\x3A\x220.0439084991812706\x22,\x22player\x22\x3A\x22David\x20McGoldrick\x22,\x22h_a\x22\x3A\x22a\x22,\x22player_id\x22\x3A\x227711\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22RightFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22George\x20Baldock\x22,\x22lastAction\x22\x3A\x22Pass\x22\x7D,\x7B\x22id\x22\x3A\x22401355\x22,\x22minute\x22\x3A\x2273\x22,\x22result\x22\x3A\x22BlockedShot\x22,\x22X\x22\x3A\x220.875\x22,\x22Y\x22\x3A\x220.3609999847412109\x22,\x22xG\x22\x3A\x220.07747054845094681\x22,\x22player\x22\x3A\x22Oliver\x20Burke\x22,\x22h_a\x22\x3A\x22a\x22,\x22player_id\x22\x3A\x225256\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22RightFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22John\x20Lundstram\x22,\x22lastAction\x22\x3A\x22Pass\x22\x7D,\x7B\x22id\x22\x3A\x22401356\x22,\x22minute\x22\x3A\x2273\x22,\x22result\x22\x3A\x22Goal\x22,\x22X\x22\x3A\x220.8769999694824219\x22,\x22Y\x22\x3A\x220.3079999923706055\x22,\x22xG\x22\x3A\x220.28404054045677185\x22,\x22player\x22\x3A\x22Oliver\x20Burke\x22,\x22h_a\x22\x3A\x22a\x22,\x22player_id\x22\x3A\x225256\x22,\x22situation\x22\x3A\x22OpenPlay\x22,\x22season\x22\x3A\x222020\x22,\x22shotType\x22\x3A\x22RightFoot\x22,\x22match_id\x22\x3A\x2214628\x22,\x22h_team\x22\x3A\x22Manchester\x20United\x22,\x22a_team\x22\x3A\x22Sheffield\x20United\x22,\x22h_goals\x22\x3A\x221\x22,\x22a_goals\x22\x3A\x222\x22,\x22date\x22\x3A\x222021\x2D01\x2D27\x2020\x3A15\x3A00\x22,\x22player_assisted\x22\x3A\x22John\x20Lundstram\x22,\x22lastAction\x22\x3A\x22Pass\x22\x7D\x5D\x7D')

Can anyone help me debug the code so I can fetch the data in JSON format?

Improve this question
2
  • can you print json_data, putting its content between delimiters of your choice, like print("**{json_data}**") just to see if it starts or is followed by line returns or whitespace Commented Jan 28, 2021 at 7:49
  • 1
    Please see I have updated the question with complete strings of shots and I am using "('"+1 because that's my starting index to fetch the data. @JulesG.M. Commented Jan 28, 2021 at 8:06

1 Answer 1

Reset to default
1

The problem is your json_data as a string starts with the '{. The start index you want is actually one more index value ahead at the {, so you want to add 2, not 1 to the index start:

index_start = strings.index("('")+2 instead of index_start = strings.index("('")+1

#Import packages and modules
import requests
from bs4 import BeautifulSoup
import json
import pandas as pd

#Set up the url and class to scrape
#scrape the single game shot
base_url = "https://understat.com/match/"

match_id = str(input('Please enter the match id: '))
#match_id = '14628'
url = base_url + match_id

print(url)

res = requests.get(url)
soup = BeautifulSoup(res.content, 'lxml')
scripts = soup.find_all('script')

print(scripts)

#only the shot data
strings = scripts[1].string
print(strings)

#strip symbols so we only have the json data
index_start = strings.index("('")+2   # <--- CHANGED HERE
index_end = strings.index("')")

json_data = strings[index_start:index_end]
json_data = json_data.encode('utf8').decode('unicode_escape')

#convert string to json format
data = json.loads(json_data)
print(data)

Output:

{'h': [{'id': '401340', 'minute': '10', 'result': 'MissedShots', 'X': '0.8390000152587891', 'Y': '0.5379999923706055', 'xG': '0.08287161588668823', 'player': 'Marcus Rashford', 'h_a': 'h', 'player_id': '556', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'RightFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'Aaron Wan-Bissaka', 'lastAction': 'Pass'}, {'id': '401342', 'minute': '16', 'result': 'BlockedShot', 'X': '0.9230000305175782', 'Y': '0.705999984741211', 'xG': '0.04452449828386307', 'player': 'Mason Greenwood', 'h_a': 'h', 'player_id': '7490', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'LeftFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'Bruno Fernandes', 'lastAction': 'Chipped'}, {'id': '401344', 'minute': '26', 'result': 'SavedShot', 'X': '0.74', 'Y': '0.37900001525878907', 'xG': '0.019290726631879807', 'player': 'Mason Greenwood', 'h_a': 'h', 'player_id': '7490', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'LeftFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'Aaron Wan-Bissaka', 'lastAction': 'Pass'}, {'id': '401345', 'minute': '27', 'result': 'BlockedShot', 'X': '0.8930000305175781', 'Y': '0.34900001525878904', 'xG': '0.07055725157260895', 'player': 'Mason Greenwood', 'h_a': 'h', 'player_id': '7490', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'RightFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'Aaron Wan-Bissaka', 'lastAction': 'Pass'}, {'id': '401346', 'minute': '29', 'result': 'MissedShots', 'X': '0.919000015258789', 'Y': '0.46599998474121096', 'xG': '0.04169069603085518', 'player': 'Harry Maguire', 'h_a': 'h', 'player_id': '1687', 'situation': 'FromCorner', 'season': '2020', 'shotType': 'Head', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': None, 'lastAction': 'Foul'}, {'id': '401347', 'minute': '38', 'result': 'BlockedShot', 'X': '0.7169999694824218', 'Y': '0.4370000076293945', 'xG': '0.016049593687057495', 'player': 'Bruno Fernandes', 'h_a': 'h', 'player_id': '1228', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'RightFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'Paul Pogba', 'lastAction': 'Pass'}, {'id': '401348', 'minute': '40', 'result': 'SavedShot', 'X': '0.9330000305175781', 'Y': '0.485', 'xG': '0.0721256285905838', 'player': 'Anthony Martial', 'h_a': 'h', 'player_id': '553', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'Head', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'Bruno Fernandes', 'lastAction': 'Pass'}, {'id': '401349', 'minute': '48', 'result': 'MissedShots', 'X': '0.925', 'Y': '0.35700000762939454', 'xG': '0.27813461422920227', 'player': 'Mason Greenwood', 'h_a': 'h', 'player_id': '7490', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'RightFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'Bruno Fernandes', 'lastAction': 'Pass'}, {'id': '401350', 'minute': '51', 'result': 'BlockedShot', 'X': '0.769000015258789', 'Y': '0.35200000762939454', 'xG': '0.019760465249419212', 'player': 'Aaron Wan-Bissaka', 'h_a': 'h', 'player_id': '5584', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'RightFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': None, 'lastAction': 'None'}, {'id': '401352', 'minute': '61', 'result': 'SavedShot', 'X': '0.6830000305175781', 'Y': '0.5070000076293946', 'xG': '0.032827332615852356', 'player': 'Marcus Rashford', 'h_a': 'h', 'player_id': '556', 'situation': 'DirectFreekick', 'season': '2020', 'shotType': 'RightFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': None, 'lastAction': 'Standard'}, {'id': '401353', 'minute': '63', 'result': 'BlockedShot', 'X': '0.77', 'Y': '0.6519999694824219', 'xG': '0.061657458543777466', 'player': 'Bruno Fernandes', 'h_a': 'h', 'player_id': '1228', 'situation': 'DirectFreekick', 'season': '2020', 'shotType': 'RightFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': None, 'lastAction': 'Standard'}, {'id': '401354', 'minute': '63', 'result': 'Goal', 'X': '0.9180000305175782', 'Y': '0.51', 'xG': '0.04361902177333832', 'player': 'Harry Maguire', 'h_a': 'h', 'player_id': '1687', 'situation': 'FromCorner', 'season': '2020', 'shotType': 'Head', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'Alex Telles', 'lastAction': 'Aerial'}, {'id': '401357', 'minute': '76', 'result': 'BlockedShot', 'X': '0.7609999847412109', 'Y': '0.6780000305175782', 'xG': '0.016254646703600883', 'player': 'Alex Telles', 'h_a': 'h', 'player_id': '1828', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'LeftFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': None, 'lastAction': 'None'}, {'id': '401358', 'minute': '76', 'result': 'MissedShots', 'X': '0.924000015258789', 'Y': '0.5070000076293946', 'xG': '0.10926949232816696', 'player': 'Paul Pogba', 'h_a': 'h', 'player_id': '1740', 'situation': 'FromCorner', 'season': '2020', 'shotType': 'Head', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'Alex Telles', 'lastAction': 'Cross'}, {'id': '401359', 'minute': '81', 'result': 'MissedShots', 'X': '0.8630000305175781', 'Y': '0.5570000076293945', 'xG': '0.06367684155702591', 'player': 'Edinson Cavani', 'h_a': 'h', 'player_id': '3294', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'LeftFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': None, 'lastAction': 'None'}, {'id': '401360', 'minute': '85', 'result': 'BlockedShot', 'X': '0.86', 'Y': '0.775', 'xG': '0.02079056017100811', 'player': 'Bruno Fernandes', 'h_a': 'h', 'player_id': '1228', 'situation': 'FromCorner', 'season': '2020', 'shotType': 'RightFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'Luke Shaw', 'lastAction': 'Pass'}], 'a': [{'id': '401341', 'minute': '15', 'result': 'SavedShot', 'X': '0.8690000152587891', 'Y': '0.5479999923706055', 'xG': '0.12690183520317078', 'player': 'Billy Sharp', 'h_a': 'a', 'player_id': '7712', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'LeftFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'John Fleck', 'lastAction': 'Throughball'}, {'id': '401343', 'minute': '22', 'result': 'Goal', 'X': '0.98', 'Y': '0.49400001525878906', 'xG': '0.15722613036632538', 'player': 'Kean Bryan', 'h_a': 'a', 'player_id': '9163', 'situation': 'FromCorner', 'season': '2020', 'shotType': 'Head', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'John Fleck', 'lastAction': 'Cross'}, {'id': '401351', 'minute': '55', 'result': 'MissedShots', 'X': '0.8430000305175781', 'Y': '0.29600000381469727', 'xG': '0.0439084991812706', 'player': 'David McGoldrick', 'h_a': 'a', 'player_id': '7711', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'RightFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'George Baldock', 'lastAction': 'Pass'}, {'id': '401355', 'minute': '73', 'result': 'BlockedShot', 'X': '0.875', 'Y': '0.3609999847412109', 'xG': '0.07747054845094681', 'player': 'Oliver Burke', 'h_a': 'a', 'player_id': '5256', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'RightFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'John Lundstram', 'lastAction': 'Pass'}, {'id': '401356', 'minute': '73', 'result': 'Goal', 'X': '0.8769999694824219', 'Y': '0.3079999923706055', 'xG': '0.28404054045677185', 'player': 'Oliver Burke', 'h_a': 'a', 'player_id': '5256', 'situation': 'OpenPlay', 'season': '2020', 'shotType': 'RightFoot', 'match_id': '14628', 'h_team': 'Manchester United', 'a_team': 'Sheffield United', 'h_goals': '1', 'a_goals': '2', 'date': '2021-01-27 20:15:00', 'player_assisted': 'John Lundstram', 'lastAction': 'Pass'}]}

Also you could replace:

#strip symbols so we only have the json data
index_start = strings.index("('")+2   # <--- CHANGED HERE
index_end = strings.index("')")

json_data = strings[index_start:index_end]

with:

#strip symbols so we only have the json data
json_data = strings.split("('", 1)[-1].rsplit("')",2)[0]
Improve this answer
Sign up to request clarification or add additional context in comments.

Comments

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.