Revolutionizing Chatbots: The Power of Amazon Lex and Bedrock Combined

				
					import boto3
import json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse

def get_links_within_domain(url):
    # Fetch the HTML content of the URL
    response = requests.get(url)
    if response.status_code != 200:
        print(f"Failed to retrieve the URL: {url}")
        return []
    # Parse the HTML content using BeautifulSoup
    soup = BeautifulSoup(response.text, 'html.parser')
    # Extract the base domain to ensure we only collect links within the same domain
    base_domain = urlparse(url).netloc
    # Find all links within the same domain
    links = set()
    for a_tag in soup.find_all('a', href=True):
        link = a_tag['href']
        # Resolve relative URLs
        full_url = urljoin(url, link)
        # Check if the link is within the same domain
        if urlparse(full_url).netloc == base_domain:
            links.add(full_url)
    # Remove links where the last phrase starts with #
    cleaned_links = [link for link in links if not link.split('/')[-1].startswith('#')]
    return cleaned_links

def lambda_handler(event, context):
    s3_client = boto3.client('s3')
    bucket_name = event['bucket_name']
    base_url = event['base_url']

    links = get_links_within_domain(base_url)
    
    for link in links:
        try:
            page_response = requests.get(link)
            page_response.raise_for_status()
            page_soup = BeautifulSoup(page_response.content, 'html.parser')
            page_content = page_soup.prettify()

            file_name = "homepage.html" if link == base_url else link.rstrip('/').split('/')[-1] + ".html"
            s3_client.put_object(Bucket=bucket_name, Key=file_name, Body=page_content)
            
        except requests.exceptions.RequestException as e:
            return {"errorMessage": f"Failed to fetch {link}: {e}"}
        except Exception as e:
            return {"errorMessage": f"Failed to upload {link}: {e}"}
                
    return {"message": "Files uploaded successfully."}