How to Scrape TripAdvisor Hotel Data using Python

                                      def get_driver_object():
    """
    Creates and returns the selenium webdriver object 

    Returns:
        Chromedriver object: This driver object can be used to simulate the webbrowser
    """
    
    # Creating the service object to pass the executable chromedriver path to webdriver
    service_object = Service(executable_path=CHROME_DRIVER_PATH)
    
    # Creating the ChromeOptions object to pass the additional arguments to webdriver
    options = webdriver.ChromeOptions()
    
    # Adding the arguments to ChromeOptions object
    options.headless = True                    #To run the chrome without GUI
    options.add_argument("start-maximized")      #To start the window maximised 
    options.add_argument("--disable-extensions") #To disable all the browser extensions
    options.add_argument("--log-level=3")        #To to capture the logs from level 3 or above
    options.add_experimental_option(
        "prefs", {"profile.managed_default_content_settings.images": 2}
    )                                           #To disable the images that are loaded when the website is opened
    
    
    # Creating the Webdriver object of type Chrome by passing service and options arguments
    driver_object = webdriver.Chrome(service=service_object,options=options)
    
    
    return driver_object

def get_website_driver(driver=get_driver_object(),url=SCRAPING_URL):
    """it will get the chromedriver object and opens the given URL

    Args:
        driver (Chromedriver): _description_. Defaults to get_driver_object().
        url (str, optional): URL of the website. Defaults to SCRAPING_URL.

    Returns:
        Chromedriver: The driver where the given url is opened.
    """

    # Opening the URL with the created driver object
    print("The webdriver is created") 
    driver.get(url)
    print(f"The URL '{url}' is opened")
    return driver

def search_hotels(driver):
    """Opens the Hotels page from which the data can be parsed.

    Args:
        driver (Chromedriver): The driver where the url is opened.
    """
    #Opening the Hotels tab with the given city and waiting for it to load
    open_hotels_tab(driver)
    time.sleep(10)
    
    #Selecting the Check In and Check Out Dates
    select_check_in_out_dates(driver)
    
    #Updating the details
    update_button(driver)
    time.sleep(10)

def parse_hotels(driver):
    """ To parse th web page using the BeautifulSoup

    Args:
        driver (Chromedriver): The driver instance where the hotel details are loaded
    """
    # Getting the HTML page source
    html_source = driver.page_source

    # Creating the BeautifulSoup object with the html source
    soup = BeautifulSoup(html_source,"html.parser")
    
    # Finding all the Hotel Div's in the BeautifulSoup object 
    hotel_tags = soup.find_all("div",{"data-prwidget-name":"meta_hsx_responsive_listing"})
    
    # Parsing the hotel details 
    for hotel in hotel_tags:
        # condition to check if the hotel is sponsered, ignore this hotel if it is sponsered
        sponsered = False if hotel.find("span",class_="ui_merchandising_pill") is None else True
        if not sponsered:
            parse_hotel_details(hotel)
    print("The Hotels details in the current page are parsed")

def parse_hotel_details(hotel):
    """Parse the hotel details from the given hotel div element

    Args:
        hotel (Beautifulsoup object): The hotel div element which contains hotel details
    """
    #declaring the global variables
    global HOTELS_LIST

    #Parsing the best offer Hotel Details
    best_offer_deals = parse_best_offer(hotel)
    
    #Parsing the other offers Hotel Details 
    hotel_details = parse_other_offers(hotel,best_offer_deals)
    
    # Apending the data to the hotels list
    HOTELS_LIST.append(hotel_details)

def write_to_csv():
    """To Write the hotels data in to a CSV file using pandas
    """
    #declaring the global variables
    global HOTELS_LIST,HOTELS_DF

    # Creating the pandas DataFrame object
    HOTELS_DF = pd.DataFrame(HOTELS_LIST,index=None)

    # Viewing the DataFrame
    print(f"The number of columns parsed is {HOTELS_DF.shape[1]}")
    print(f"The number of rows parsed is {HOTELS_DF.shape[0]}")

    # Conveting the DataFrame to CSV file
    HOTELS_DF.to_csv("hotels_list.csv",index=False)
    print("The CSV file is created at hotels_list.csv")

hotels_csv_file = pd.read_csv("hotels_list.csv")
hotels_csv_file.head()