diff --git a/.gitignore b/.gitignore index 401ba58..3742713 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,15 @@ celerybeat-schedule.* src/celerybeat-schedule.* feed*.json -.DS_Store \ No newline at end of file +.DS_Store + +# Ignore all __pycache__ directories at any level +**/__pycache__/ + +# Ignore all compiled Python files +*.pyc +*.pyo +*.pyd + +#Ignore saved data +data/* \ No newline at end of file diff --git a/bbc_world.xml b/bbc_world.xml deleted file mode 100644 index 35a7e7e..0000000 --- a/bbc_world.xml +++ /dev/null @@ -1,242 +0,0 @@ - - - <![CDATA[BBC News]]> - - https://www.bbc.co.uk/news/world - - https://news.bbcimg.co.uk/nol/shared/img/bbc_news_120x60.gif - BBC News - https://www.bbc.co.uk/news/world - - RSS for Node - Mon, 23 Jun 2025 16:59:28 GMT - - - - 15 - - <![CDATA[Israel says it struck Tehran's Evin prison and Fordo access routes]]> - - https://www.bbc.com/news/articles/cp8621gnknjo - https://www.bbc.com/news/articles/cp8621gnknjo#0 - Mon, 23 Jun 2025 16:29:51 GMT - - - - <![CDATA['Why wouldn't there be a regime change?' Trump's latest on Iran]]> - - https://www.bbc.com/news/articles/cp8m3861637o - https://www.bbc.com/news/articles/cp8m3861637o#0 - Mon, 23 Jun 2025 13:01:31 GMT - - - - <![CDATA[Bride shot dead in attack on French wedding party]]> - - https://www.bbc.com/news/articles/cn0q8pypxwxo - https://www.bbc.com/news/articles/cn0q8pypxwxo#0 - Mon, 23 Jun 2025 00:50:23 GMT - - - - <![CDATA[Nine killed in Kyiv in intense Russian air attack]]> - - https://www.bbc.com/news/articles/c0k7vll16l3o - https://www.bbc.com/news/articles/c0k7vll16l3o#0 - Mon, 23 Jun 2025 13:12:01 GMT - - - - <![CDATA[Scottish wingsuit flyer Liam Byrne dies during Swiss Alps jump]]> - - https://www.bbc.com/news/articles/cwyepy2zz11o - https://www.bbc.com/news/articles/cwyepy2zz11o#0 - Mon, 23 Jun 2025 13:36:18 GMT - - - - <![CDATA[Rescuers search for Brazilian tourist on Indonesian volcano]]> - - https://www.bbc.com/news/articles/crk60k8gp8jo - https://www.bbc.com/news/articles/crk60k8gp8jo#0 - Mon, 23 Jun 2025 11:10:36 GMT - - - - <![CDATA[First celestial image unveiled from revolutionary telescope]]> - - https://www.bbc.com/news/articles/cj3rmjjgx6xo - https://www.bbc.com/news/articles/cj3rmjjgx6xo#1 - Mon, 23 Jun 2025 16:35:11 GMT - - - - <![CDATA[Nato summit could be the most significant since end of Cold War]]> - - https://www.bbc.com/news/articles/cqjqvr75v1jo - https://www.bbc.com/news/articles/cqjqvr75v1jo#1 - Mon, 23 Jun 2025 14:02:57 GMT - - - - <![CDATA[Police officers charged with murder of Kenyan blogger]]> - - https://www.bbc.com/news/articles/cz09l4k4184o - https://www.bbc.com/news/articles/cz09l4k4184o#1 - Mon, 23 Jun 2025 15:46:31 GMT - - - - <![CDATA[Dozens of soldiers seized by locals in Colombian mountains]]> - - https://www.bbc.com/news/articles/c4gdlrk1pzko - https://www.bbc.com/news/articles/c4gdlrk1pzko#1 - Mon, 23 Jun 2025 04:09:05 GMT - - - - <![CDATA[Belarus opposition leader's husband urges Trump to help other prisoners ]]> - - https://www.bbc.com/news/articles/c07dgmymd1eo - https://www.bbc.com/news/articles/c07dgmymd1eo#1 - Sun, 22 Jun 2025 17:04:56 GMT - - - - <![CDATA[Suicide bombing at Damascus church kills 25, Syrian authorities say]]> - - https://www.bbc.com/news/articles/c307n9p43z9o - https://www.bbc.com/news/articles/c307n9p43z9o#1 - Sun, 22 Jun 2025 21:26:21 GMT - - - - <![CDATA[Lyse Doucet: What the US and Iran do next could be momentous]]> - - https://www.bbc.com/news/articles/cy0w94yj68xo - https://www.bbc.com/news/articles/cy0w94yj68xo#2 - Sun, 22 Jun 2025 21:41:06 GMT - - - - <![CDATA[Decoy flights and seven B-2 stealth bombers - how US says it hit Iran's nuclear sites]]> - - https://www.bbc.com/news/articles/cew0x7159edo - https://www.bbc.com/news/articles/cew0x7159edo#2 - Sun, 22 Jun 2025 21:45:53 GMT - - - - <![CDATA[How China made electric vehicles mainstream]]> - - https://www.bbc.com/news/articles/c2d5ld8y8pwo - https://www.bbc.com/news/articles/c2d5ld8y8pwo#2 - Sun, 22 Jun 2025 23:17:26 GMT - - - - <![CDATA[Caught in the crossfire - the victims of Cape Town's gang warfare]]> - - https://www.bbc.com/news/articles/c1mg2xg339jo - https://www.bbc.com/news/articles/c1mg2xg339jo#2 - Sun, 22 Jun 2025 23:57:19 GMT - - - - <![CDATA['Tourists are our lifeline': Picking up the pieces in Kashmir after April killings]]> - - https://www.bbc.com/news/articles/cly8lx75d6ro - https://www.bbc.com/news/articles/cly8lx75d6ro#2 - Sun, 22 Jun 2025 23:53:10 GMT - - - - <![CDATA[What we know about US strikes on three Iranian nuclear sites]]> - - https://www.bbc.com/news/articles/cvg9r4q99g4o - https://www.bbc.com/news/articles/cvg9r4q99g4o#2 - Mon, 23 Jun 2025 14:29:31 GMT - - - - <![CDATA['Paralysed and constantly caught by surprise': Voices from Iran after US strikes]]> - - https://www.bbc.com/news/articles/cpwq2vnd827o - https://www.bbc.com/news/articles/cpwq2vnd827o#2 - Sun, 22 Jun 2025 16:25:21 GMT - - - - <![CDATA[How a self-styled knight giving away cars and wads of cash got people talking]]> - - https://www.bbc.com/news/articles/cn81ynyjy8no - https://www.bbc.com/news/articles/cn81ynyjy8no#2 - Sun, 22 Jun 2025 01:31:32 GMT - - - - <![CDATA[Watch: Video shows aftermath of Israeli strikes on Iran]]> - - https://www.bbc.com/news/videos/c8xvqje77ygo - https://www.bbc.com/news/videos/c8xvqje77ygo#3 - Mon, 23 Jun 2025 16:44:46 GMT - - - - <![CDATA[Netanyahu and Trump 'totally aligned' on Iran - Barkat]]> - - https://www.bbc.com/news/videos/c3en18eq435o - https://www.bbc.com/news/videos/c3en18eq435o#3 - Mon, 23 Jun 2025 15:31:40 GMT - - - - <![CDATA[Watch: How successful have the US strikes on Iran been?]]> - - https://www.bbc.com/news/videos/cq53l9dvggjo - https://www.bbc.com/news/videos/cq53l9dvggjo#3 - Sun, 22 Jun 2025 20:39:06 GMT - - - - <![CDATA[US strikes on Iran trigger protests internationally]]> - - https://www.bbc.com/news/videos/c4g2wxwel5qo - https://www.bbc.com/news/videos/c4g2wxwel5qo#3 - Mon, 23 Jun 2025 01:46:57 GMT - - - - <![CDATA[Watch: Trump says strikes on Iran a 'spectacular military success' in address]]> - - https://www.bbc.com/news/videos/c93k8nvdnqpo - https://www.bbc.com/news/videos/c93k8nvdnqpo#3 - Sun, 22 Jun 2025 02:52:36 GMT - - - - <![CDATA['Necessary' or 'unfounded'? Americans in Texas respond to US strikes]]> - - https://www.bbc.com/news/videos/c07dg04ke45o - https://www.bbc.com/news/videos/c07dg04ke45o#3 - Sun, 22 Jun 2025 22:47:25 GMT - - - - <![CDATA[Satellite images and decoy planes: Behind the US attacks on Iran]]> - - https://www.bbc.com/news/videos/cdezkx5nl1wo - https://www.bbc.com/news/videos/cdezkx5nl1wo#3 - Sun, 22 Jun 2025 18:02:12 GMT - - - - <![CDATA[Watch: The US has joined the Iran-Israel war. What happens now?]]> - - https://www.bbc.com/news/videos/c5ypw09gdzpo - https://www.bbc.com/news/videos/c5ypw09gdzpo#3 - Sun, 22 Jun 2025 13:58:09 GMT - - - - \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4382f06..8d43ec9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,9 +28,36 @@ services: dockerfile: .docker/Dockerfile volumes: - .:/workspace:cached + - mongo_data:/data/db depends_on: - redis + - mongo #entrypoint: ["/usr/local/bin/entrypoint.sh"] command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important + celery_beat: + build: + context: . + dockerfile: .docker/Dockerfile + volumes: + - .:/workspace:cached + depends_on: + - redis + #entrypoint: ["/usr/local/bin/entrypoint.sh"] + command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important + + mongo: + image: mongo:latest + container_name: mongo + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: myuser + MONGO_INITDB_ROOT_PASSWORD: mypassword + ports: + - "27017:27017" + volumes: + - mongo_data:/data/db + - .:/workspace:cached +volumes: + mongo_data: \ No newline at end of file diff --git a/readme.md b/readme.md index c52820d..fa61a1f 100644 --- a/readme.md +++ b/readme.md @@ -22,7 +22,7 @@ A lightweight, news intelligence service for ingesting free news sources, taggin cd justinsight -## Setting up Celery Beat: +## Setting up Celery Beat - On MacOS - run: docker compose up --build - OR for running in the background: docker compose up -d @@ -34,5 +34,15 @@ A lightweight, news intelligence service for ingesting free news sources, taggin - use EC2 section of entrypoint.sh - to shut down celery: docker compose down -*When running in background you can use "docker logs " to see the log and "docker ps" -to find the ID. \ No newline at end of file +Note: When running in background you can use "docker logs " to see the log and "docker ps" to find the ID. + +## How to Create and Schedule a Task +1) Create a method for your task wherever you are working on it that can be called and complete the task. It should be in a package (ie there should be an \_\_init\_\_.py file in the same folder as the file you are working on). +2) Go to src/justinsight/tasks.py and create a new task following the format of the sample task. Remember to add an import statement if one is needed. +3) Go to src/justinsight/celery.py and schedule a new task after the comment showing where to schedule them. The new task can follow the same format as those above + + "sample-task-name": { + "task": "justinsight.tasks.your_task_name", + "schedule": x, #where x is the number of seconds between when the task should happen + "args": (), #potential arguments for your task + }, \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1ce978e..18115a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,6 @@ -celery[redis] \ No newline at end of file +beautifulsoup4 +celery[redis] +feedparser +playwright +pymongo +requests \ No newline at end of file diff --git a/src/ingest/__pycache__/__init__.cpython-310.pyc b/src/ingest/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 5817e24..0000000 Binary files a/src/ingest/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/src/ingest/__pycache__/bbc_rss.cpython-310.pyc b/src/ingest/__pycache__/bbc_rss.cpython-310.pyc deleted file mode 100644 index acc5de4..0000000 Binary files a/src/ingest/__pycache__/bbc_rss.cpython-310.pyc and /dev/null differ diff --git a/src/ingest/base_ingestor.py b/src/ingest/base_ingestor.py new file mode 100644 index 0000000..0c5eaee --- /dev/null +++ b/src/ingest/base_ingestor.py @@ -0,0 +1,52 @@ +import datetime +import feedparser +import hashlib +import re +from src.ingest.save_to_database import save_entry + +class BaseIngestor: + RSS_URL = None #will be set by the subclasses + + def slugify(self, text): + # Convert title to a filesystem-friendly slug + text = text.lower() + text = re.sub(r'[^a-z0-9]+', '-', text) + return text.strip('-') + + def format_date(self, entry): + # Extract and format the published date + try: + dt = datetime.datetime(*entry.published_parsed[:6]) + return dt.strftime("%Y-%m-%d") + except: + return "unknown-date" + + def fetch_full_text(self, url): + raise NotImplementedError( + "Subclass must implement fetch_full_text()" + ) + + def generate_entry_hash(self, entry): + hash_input = f"{entry.title}{entry.link}{entry.get('published', '')}" + return hashlib.sha256(hash_input.encode('utf-8')).hexdigest() + + def format_entry(self, entry): + title_slug = self.slugify(entry.title) + date_str = self.format_date(entry) + full_text = self.fetch_full_text(entry.link) + + data = { + "title": entry.title, + "link": entry.link, + "published": entry.get("published", ""), + "summary": entry.get("summary", ""), + "full_text": full_text, + "id": self.generate_entry_hash(entry) + } + return data + + def check_and_save_new_entries(self): + feed = feedparser.parse(self.RSS_URL) + + for entry in feed.entries: + save_entry(self.format_entry(entry)) diff --git a/src/ingest/bbc_ingestor.py b/src/ingest/bbc_ingestor.py new file mode 100644 index 0000000..71d5542 --- /dev/null +++ b/src/ingest/bbc_ingestor.py @@ -0,0 +1,15 @@ +from bs4 import BeautifulSoup +import requests +from src.ingest.base_ingestor import BaseIngestor + +class BBCIngestor(BaseIngestor): + RSS_URL = "http://feeds.bbci.co.uk/news/world/rss.xml" + + def fetch_full_text(self, article_url): + response = requests.get(self.RSS_URL) + soup = BeautifulSoup(response.content, 'html.parser') + + article = soup.find('article') + + if article: + return(article.get_text()) \ No newline at end of file diff --git a/src/ingest/save_to_database.py b/src/ingest/save_to_database.py new file mode 100644 index 0000000..3feb6f7 --- /dev/null +++ b/src/ingest/save_to_database.py @@ -0,0 +1,18 @@ +from pymongo import MongoClient + +# Include username, password, and authentication database +client = MongoClient("mongodb://myuser:mypassword@mongo:27017/justinsightdb?authSource=admin") + +# Get (or create) a database +db = client["justinsightdb"] + +# Get (or create) a collection +collection = db["articles"] + +def save_entry(entry): + #check if the entry has already been saved and if it has not then save it + entry_hash = entry["id"] + if collection.count_documents({"id": entry_hash}) == 0: + collection.insert_one(entry) + else: + print(f"{entry['title']} already in use!") diff --git a/src/justInsight/__pycache__/__init__.cpython-310.pyc b/src/justInsight/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 3dee5e1..0000000 Binary files a/src/justInsight/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/src/justInsight/__pycache__/celery.cpython-310.pyc b/src/justInsight/__pycache__/celery.cpython-310.pyc deleted file mode 100644 index 7fa8fd6..0000000 Binary files a/src/justInsight/__pycache__/celery.cpython-310.pyc and /dev/null differ diff --git a/src/justInsight/__pycache__/tasks.cpython-310.pyc b/src/justInsight/__pycache__/tasks.cpython-310.pyc deleted file mode 100644 index 81b5e86..0000000 Binary files a/src/justInsight/__pycache__/tasks.cpython-310.pyc and /dev/null differ diff --git a/src/justInsight/celery.py b/src/justInsight/celery.py deleted file mode 100644 index 5a92454..0000000 --- a/src/justInsight/celery.py +++ /dev/null @@ -1,18 +0,0 @@ -# celery.py -from celery import Celery - -app = Celery( - "justInsight", - broker="redis://redis:6379/0", - backend="redis://redis:6379/0", - include=["justInsight.tasks"], # adjust this to your tasks file -) - -# Optional beat schedule -app.conf.beat_schedule = { - "sample-task-every-10-seconds": { - "task": "justInsight.tasks.sample_task", - "schedule": 10.0, - "args": (), - }, -} diff --git a/src/justInsight/tasks.py b/src/justInsight/tasks.py deleted file mode 100644 index 9378f65..0000000 --- a/src/justInsight/tasks.py +++ /dev/null @@ -1,9 +0,0 @@ -# tasks.py -from celery import shared_task -from ingest.bbc_rss import testTask - - -@shared_task -def sample_task(): - testTask() - return "done!" \ No newline at end of file diff --git a/src/justInsight/__init__.py b/src/justinsight/__init__.py similarity index 100% rename from src/justInsight/__init__.py rename to src/justinsight/__init__.py diff --git a/src/justinsight/celery.py b/src/justinsight/celery.py new file mode 100644 index 0000000..b5b241a --- /dev/null +++ b/src/justinsight/celery.py @@ -0,0 +1,32 @@ +# celery.py +from celery import Celery + +app = Celery( + "justinsight", + broker="redis://redis:6379/0", + backend="redis://redis:6379/0", + include=["justinsight.tasks"], +) + +# Optional beat schedule +app.conf.beat_schedule = { + # "sample-task-every-5-seconds": { + # "task": "justinsight.tasks.sample_task", + # "schedule": 5.0, + # "args": (), + # }, + + "check-BBCfeed-every-5-minutes": { + "task": "justinsight.tasks.bbcLogger_task", + "schedule": 5.0, + "args": (), + }, + + # "check-NYTfeed-every-5-minutes": { + # "task": "justinsight.tasks.nytLogger_task", + # "schedule": 300.0, + # "args": (), + # }, + + #schedule more tasks here +} diff --git a/src/justinsight/tasks.py b/src/justinsight/tasks.py new file mode 100644 index 0000000..7066141 --- /dev/null +++ b/src/justinsight/tasks.py @@ -0,0 +1,23 @@ +# tasks.py +from celery import shared_task +from ingest.bbc_ingestor import BBCIngestor + + +@shared_task +def sample_task(): + print("Sample task!") + return "done!" + +@shared_task +def bbcLogger_task(): + # Create an instance of the class + ingestor = BBCIngestor() + ingestor.check_and_save_new_entries() # this will invoke the inherited logic + return "BBC RSS Feed checked." + +@shared_task +def nytLogger_task(): + check_and_save_nyt() + return "NYT RSS Feed checked." + +#Add more tasks here in the format of the one above \ No newline at end of file