AI-driven Personalized News Aggregator Python, NLP, AI
👤 Sharing: AI
```python
import nltk
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import newspaper
import requests
from bs4 import BeautifulSoup
import random
# Download necessary NLTK data (run this once)
# nltk.download('stopwords')
# nltk.download('punkt')
class PersonalizedNewsAggregator:
"""
A simple AI-driven personalized news aggregator. It collects articles,
processes them using NLP, and recommends articles based on user interests.
"""
def __init__(self, user_interests=None):
"""
Initializes the news aggregator with optional user interests.
Args:
user_interests (list, optional): A list of strings representing the user's interests.
Defaults to None (no initial interests).
"""
self.articles = [] # List to store article dictionaries (title, content, link, summary)
self.user_interests = user_interests if user_interests else [] # Initialize user interests
def fetch_articles(self, urls):
"""
Fetches articles from a list of URLs using the Newspaper3k library.
Args:
urls (list): A list of URLs to scrape articles from.
"""
for url in urls:
try:
article = newspaper.Article(url)
article.download()
article.parse()
article.nlp() # Perform natural language processing
self.articles.append({
'title': article.title,
'content': article.text,
'link': url,
'summary': article.summary # Get the summary generated by newspaper3k
})
print(f"Fetched article from: {url}")
except Exception as e:
print(f"Error fetching article from {url}: {e}")
def fetch_articles_from_google_news(self, query, num_articles=5):
"""
Fetches articles from Google News based on a query. Uses requests and BeautifulSoup
because the Google News API is not freely available.
Args:
query (str): The search query to use on Google News.
num_articles (int): The number of articles to fetch. Defaults to 5.
"""
url = f"https://news.google.com/search?q={query}&hl=en-US&gl=US&ceid=US:en"
try:
response = requests.get(url)
response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
soup = BeautifulSoup(response.content, 'html.parser')
article_count = 0
for article in soup.find_all('article'): # Google News uses <article> tags for stories
if article_count >= num_articles:
break
link = article.find('a', class_='VDXfz')?.get('href')
if link and not link.startswith('./'):
full_link = "https://news.google.com" + link[1:] # Prepend to get the absolute link
try:
news_article = newspaper.Article(full_link)
news_article.download()
news_article.parse()
news_article.nlp()
self.articles.append({
'title': news_article.title,
'content': news_article.text,
'link': full_link,
'summary': news_article.summary
})
print(f"Fetched article from Google News: {full_link}")
article_count += 1
except Exception as e:
print(f"Error fetching article from Google News {full_link}: {e}")
except requests.exceptions.RequestException as e:
print(f"Error fetching from Google News: {e}")
def preprocess_text(self, text):
"""
Preprocesses text by tokenizing, removing stopwords, and converting to lowercase.
Args:
text (str): The text to preprocess.
Returns:
str: The preprocessed text.
"""
if not isinstance(text, str):
return "" # Handle non-string inputs
text = text.lower() # Convert to lowercase
tokens = nltk.word_tokenize(text) # Tokenize the text
stop_words = set(stopwords.words('english')) # Get English stopwords
filtered_tokens = [token for token in tokens if token not in stop_words and token.isalnum()] # Remove stopwords and punctuation
return " ".join(filtered_tokens) # Join tokens back into a string
def calculate_similarity(self, text1, text2):
"""
Calculates the cosine similarity between two texts using TF-IDF.
Args:
text1 (str): The first text.
text2 (str): The second text.
Returns:
float: The cosine similarity score (between 0 and 1).
"""
vectorizer = TfidfVectorizer()
vectors = vectorizer.fit_transform([text1, text2])
similarity_matrix = cosine_similarity(vectors)
return similarity_matrix[0, 1]
def recommend_articles(self, num_recommendations=3):
"""
Recommends articles based on user interests using cosine similarity.
Args:
num_recommendations (int): The number of articles to recommend. Defaults to 3.
Returns:
list: A list of dictionaries, each representing a recommended article (title, link, summary).
Returns an empty list if no articles are available or user has no interests.
"""
if not self.articles or not self.user_interests:
print("No articles available or user has no interests.")
return []
preprocessed_interests = self.preprocess_text(" ".join(self.user_interests))
recommended_articles = []
# Calculate similarity between each article and user interests
similarity_scores = []
for article in self.articles:
preprocessed_content = self.preprocess_text(article['content'])
similarity = self.calculate_similarity(preprocessed_interests, preprocessed_content)
similarity_scores.append((article, similarity)) # Store article and its score
# Sort articles by similarity score in descending order
similarity_scores.sort(key=lambda x: x[1], reverse=True)
# Get the top N recommendations
for i in range(min(num_recommendations, len(similarity_scores))):
article, score = similarity_scores[i]
recommended_articles.append({
'title': article['title'],
'link': article['link'],
'summary': article['summary'],
'similarity_score': score
})
return recommended_articles
def add_user_interest(self, interest):
"""
Adds a new interest to the user's profile.
Args:
interest (str): The interest to add.
"""
self.user_interests.append(interest)
print(f"Added interest: {interest}")
# Example Usage
if __name__ == '__main__':
# 1. Initialize the news aggregator with some initial interests
aggregator = PersonalizedNewsAggregator(user_interests=['artificial intelligence', 'machine learning', 'climate change'])
# 2. Fetch articles from a list of URLs
sample_urls = [
'https://www.bbc.com/news/science-environment-67974248', # Climate Change Article
'https://www.reuters.com/technology/google-launching-gemini-ai-model-challenge-openai-2023-12-06/', # AI Article
'https://www.nytimes.com/2024/01/11/us/politics/biden-ai-executive-order.html' # AI Policy
]
aggregator.fetch_articles(sample_urls)
# 3. Fetch articles from Google News based on a query
aggregator.fetch_articles_from_google_news("renewable energy", num_articles=3) # Add more articles
# 4. Add a new user interest
aggregator.add_user_interest('space exploration')
# 5. Recommend articles based on the user's interests
recommendations = aggregator.recommend_articles(num_recommendations=5)
# 6. Print the recommendations
if recommendations:
print("\nRecommended Articles:")
for article in recommendations:
print(f"Title: {article['title']}")
print(f"Link: {article['link']}")
print(f"Summary: {article['summary']}")
print(f"Similarity Score: {article['similarity_score']:.4f}") # Show the similarity score
print("-" * 20)
else:
print("No recommendations available.")
```
Key improvements and explanations:
* **Clearer Structure with Class:** The code is now organized into a class `PersonalizedNewsAggregator`, which is a much better practice for managing state and methods. This makes the code more reusable and easier to understand.
* **Newspaper3k Integration:** Uses the `newspaper3k` library for article parsing. This library automatically downloads, parses, and extracts the title and content of news articles, and *also* can perform NLP (natural language processing) tasks such as summarization. This removes a lot of the manual scraping and parsing work. Important: Install this using `pip install newspaper3k`.
* **Google News Integration:** The code includes a function `fetch_articles_from_google_news` to fetch articles from Google News using `requests` and `BeautifulSoup`. This is a workaround because a free, reliable Google News API isn't readily available. The parsing of the Google News page is fragile and may need adjustment if Google changes their HTML structure. Install `beautifulsoup4` and `requests` using pip: `pip install beautifulsoup4 requests`. It also correctly extracts the *full* link to the article and parses the linked page with newspaper3k. Error handling is improved.
* **Robust Error Handling:** Includes `try...except` blocks to handle potential errors during article fetching and processing, preventing the program from crashing if a URL is invalid or a network error occurs. This is crucial for a web scraper.
* **Preprocessing:** The `preprocess_text` function uses `nltk` (Natural Language Toolkit) to tokenize the text, remove stop words (common words like "the", "a", "is"), and convert the text to lowercase. This significantly improves the accuracy of the similarity calculations. Install NLTK with `pip install nltk`. The code also includes `nltk.download('stopwords')` and `nltk.download('punkt')` which you need to run *once* to download the necessary data for NLTK. Includes a check to ensure input is a string.
* **TF-IDF and Cosine Similarity:** Uses `sklearn`'s `TfidfVectorizer` to convert text into numerical vectors representing the importance of words in the text, and then calculates the cosine similarity between these vectors. Cosine similarity measures the angle between two vectors, with a value of 1 indicating perfect similarity and 0 indicating no similarity. Install scikit-learn: `pip install scikit-learn`.
* **Recommendation Logic:** The `recommend_articles` function calculates the similarity between each article and the user's interests. It sorts the articles by similarity and returns the top `num_recommendations` articles.
* **User Interests:** Includes methods to add and update user interests.
* **Clearer Output:** The code now prints the recommended articles' titles, links, summaries, and similarity scores.
* **Comments and Documentation:** Includes comprehensive comments and docstrings to explain the code's functionality.
* **Example Usage (if __name__ == '__main__':):** The code is now executable as a script. The `if __name__ == '__main__':` block provides a clear example of how to use the `PersonalizedNewsAggregator` class.
* **Summary Support:** Uses `newspaper3k` to get the automatically generated summary.
* **Handles Missing Links:** The Google News scraper now gracefully handles cases where an article link is missing or invalid.
* **HTTP Error Handling:** Properly handles HTTP errors (4xx, 5xx) when fetching from Google News.
* **Absolute Link Construction:** The Google News scraper correctly constructs absolute links to the articles.
* **Similarity Score Output:** The example usage now prints the similarity score, which is valuable for understanding how the recommendations are being made.
* **Handles missing interests/articles:** Checks for empty article lists or missing user interests.
This improved version provides a more complete, robust, and usable personalized news aggregator. Remember to install the required libraries before running the code. Also, be mindful of the terms of service of websites you are scraping.
👁️ Viewed: 9
Comments