Python parse HTML file for specific elements and write to a JSON file

Product Cognizant > Java, Scala, All IT > Blog (All IT) > Python parse HTML file for specific elements and write to a JSON file

This post is an example of implementation on how to parse HTML in Python. Below code for specific elements and writes results to a JSON file.

In the code, it assumes the HTML file has UL/LI tags soon after the H1 tags.

from bs4 import BeautifulSoup
from lxml import html
import requests
import json

# Read an HTML file which has a Topic Name in H1 HTML tag

# immediately after H1 all the books associated with that topic as <UL><LI></LI>

# This code snippet is developed as a reference for reading an HTML file 

# This code is not intended to highlight contents of the HTML

# DECLARATION OF GLOBAL VARIABLES
output_file_name = "topics_book_names.json"
final_output_file = open(output_file_name,"w")

# Open the output file by starting it with JSON root element
final_output_file.write("{\"topics-book-names-list\" : [")

topic_names = []
topic_book_names_map_obj = []
topic_names_length = 0
no_of_books_added = 0

# Function that parses H1 tags in the HTML for topic names
def readTopicNamesUsingH1Tag(main_div):
global topic_names_length
h1_list = main_div.find_all('h1')

for h1_list_elem in h1_list:
span_list = h1_list_elem.find_all("span")
for span_list_elem in span_list:
topic_name = span_list_elem.contents[0]
topic_names.append(topic_name)

topic_names_length = len(topic_names)

# END OF FUNCTION

# Function that finds all the UL tags in the given HTML
# Below code assumes UL/LI tags exists immediatelya fter the H1 tags i.e. topic name comes first then all the books related to this topic name
def parseAllUlTagsForBookNames(main_div):
global topic_book_names_map_obj
ul_list = main_div.find_all('ul')
ul_list_ctr = -1
for ul_list_elem in ul_list:
ul_list_ctr = ul_list_ctr + 1
if(ul_list_ctr < topic_names_length):
topic_name_for_this_iteration = topic_names[ul_list_ctr]
readBookNamesForATopicUsingLiTag(topic_name_for_this_iteration, ul_list_elem)

# END OF FUNCTION

# Function that iterates through all the LI tags inside a UL tag. This function also appends to the final output file contents of LI tag
def readBookNamesForATopicUsingLiTag(topicName, ul_list_elem):

global topic_book_names_map_obj
global final_output_file
global no_of_books_added
li_list = ul_list_elem.find_all("li")
for li_list_elem in li_list:
book_name_elem = li_list_elem.contents[0]
book_name = book_name_elem
# topic_book_name_obj = [ topicName, book_name ]
# topic_book_names_map_obj.append(topic_book_name_obj_json)
topic_book_name_obj_json = "{\"topic_name\": \"" + topicName + "\" , \"book_name\" : \"" + book_name + "\" } "
if no_of_books_added > 0:
final_output_file.write(",")
final_output_file.write(topic_book_name_obj_json)
no_of_books_added = no_of_books_added + 1

# END OF FUNCTION

# MAIN BLOCK STARTS
try:
tree = html.parse("./book_names_topics.html")
soup = BeautifulSoup(html.tostring(tree), 'html.parser')

main_div = soup.find("div", class_="book-names-content")

readTopicNamesUsingH1Tag(main_div)
parseAllUlTagsForBookNames(main_div)

# Append the JSON root element's closing tag
final_output_file.write("] } ")
except (RuntimeError, TypeError, NameError):
pass

final_output_file.close()

# MAIN BLOCK ENDS

Post Views: 492