LesMills/code/scrape_LM.py

# %%
import requests
from bs4 import BeautifulSoup as bs
import nltk
from nltk.stem import PorterStemmer
import re
import pandas as pd


path= 'C:\\Users\\Buster\\Documents\\LesMillsProject\data\\'

videos_list = []

# season2 = 55mins

url= "https://watch.lesmillsondemand.com/bodycombat/season:2"

html_string = requests.get(url).content

soup = bs(html_string, 'html.parser')

for a in soup.find_all('a', href=True):
    videos_list.append((a['href']))

videos_list = [i for i in videos_list if 'bodycombat' in i]

videos_list = [i for i in videos_list if 'season:2' in i]

videos_set = set(videos_list)
videos_list = list(videos_set)

bc_dict={}


for i in videos_list:
    print (i)
    if len(i.split("bodycombat-",1))>1:
        release = i.split("bodycombat-",1)[1][:2] 
    elif len(i.split("bodycomabt-",1))>1:
        release = i.split("bodycomabt-",1)[1][:2] 
    else:
        release = i.split("-bodycombat",1)[1][:2] 

    html_string = requests.get(i).content

    soup = bs(html_string, 'html.parser')

    try:

        desc = soup.find_all(attrs={"name":"description"})

        bc_dict[release] = str(desc)

    except:
        print('did not have a description: ' ,i)

print(bc_dict.keys())

for k,v in bc_dict.items():
    print(k, ':  ', len(v))

# print(bc_dict)



bc_desc_df = pd.DataFrame(list(bc_dict.items()),columns = ['release','desc']) 

# moves_dict = []
# bc_desc_df['Moves']

print(bc_desc_df.head())
bc_desc_df.to_csv(path+'bc_desc_df.csv')

print(bc_dict['76'])

# %%
release_ls = []
track_ls = []
track_name_ls = []
moves_ls = []
moves_set = set()
move_dict = dict()
track_moves_dict = dict()

porter = PorterStemmer
stopwords = ['combo', 'combos', 'combination','right', 'left', '2', '4', '3']

for release_no, desc in bc_dict.items():
    a = desc.split('Moves: ') 
    if len(a)<2:
        a=  desc.split('MOVES: ')  
    track_count = -1
    
    for i in a: 
        head, sep, tail = i.partition('\n')
        if track_count == 0:
            track_name_clean = '1A. UPPER BODY WARMUP'
        if track_count == -1:
            track_name_clean = 'NA'
        
        track_moves_ls = head.split(',')
        if len(track_moves_ls) >0:
            track_moves_ls = [i for i in track_moves_ls if 'meta' not in i]
            track_moves_ls = [i.replace(' name="description"','') for i in track_moves_ls]
            track_moves_ls = [i.replace('\r','') for i in track_moves_ls]
            for word in stopwords:
                track_moves_ls = [i.replace(word,'') for i in track_moves_ls]
            track_moves_ls = [i.replace(r'\d+','') for i in track_moves_ls]
            track_moves_ls = [i.replace('&amp','') for i in track_moves_ls]
            track_moves_ls = [i.strip() for i in track_moves_ls]
            track_moves_ls = [i.lower() for i in track_moves_ls]
            #track_moves_ls = [[porter.stem(word) for word in i] for i in track_moves_ls]

            track_count +=1
            release_ls.append(release_no)
            track_ls.append(track_count)
            track_name_ls.append(track_name_clean)
            track_moves_dict[str(release_no) + str(track_name_clean)] = track_moves_ls
            moves_ls.append(track_moves_ls) #list of lists of moves
            for move in track_moves_ls:
                if move in move_dict.keys():
                    move_dict[move].append(release_no)
                else:
                    move_dict[move] = [release_no]
            

            if track_count == 0:
                track_name_clean = '1A. UPPER BODY WARMUP'
            else:
                track_name, sep, tail = tail.partition('\n')    
                track_name_clean = track_name.replace('\r','')
                track_name_clean = track_name_clean.replace('&amp','')
                track_name_clean = track_name_clean.strip()
                track_name_clean = track_name_clean.upper()


        else:
            pass

        

release_dict = {'release':release_ls, 'track':track_ls, 'track_name':track_name_ls , 'moves_ls':moves_ls }

print(release_dict)

print(move_dict)

print('release_ls: ', len(release_ls))
print('track_ls: ', len(track_ls))
print('moves_ls: ', len(moves_ls))

release_df = pd.DataFrame(release_dict)
release_df.to_csv(path+'release_df.csv')


restructured = []

for key in move_dict:
    data_dict = {}
    for release in move_dict[key]:
        data_dict[release] = 1
    restructured.append(data_dict)

from sklearn.feature_extraction import DictVectorizer
dictvectorizer = DictVectorizer(sparse=False)
features = dictvectorizer.fit_transform(restructured)


column_values=(dictvectorizer.get_feature_names())
index_values = move_dict.keys()

move_df = pd.DataFrame(data = features,  
                  index = index_values,  
                  columns = column_values) 

#move_df = pd.DataFrame(list(move_dict.items()),columns = ['move','releases']) 
print(move_df)
move_df.to_csv(path+'move_df.csv')



restructured2 = []

for track_id in track_moves_dict:
    data_dict = {}
    for move in track_moves_dict[track_id]:
        data_dict[move] = 1
    restructured2.append(data_dict)

from sklearn.feature_extraction import DictVectorizer
dictvectorizer = DictVectorizer(sparse=False)
features = dictvectorizer.fit_transform(restructured2)


column_values=(dictvectorizer.get_feature_names())
index_values = track_moves_dict.keys()

move_by_track_df = pd.DataFrame(data = features,  
                  index = index_values,  
                  columns = column_values) 

move_by_track_df['track_type'] = (move_by_track_df.index).str[2:]

#move_df = pd.DataFrame(list(move_dict.items()),columns = ['move','releases']) 
print(move_by_track_df)
move_by_track_df.to_csv(path+'move_by_track_df.csv')