From 4caabd42cba7942fd14827793fa884d8811e15e9 Mon Sep 17 00:00:00 2001 From: Justin Madison Date: Tue, 24 Jun 2025 11:41:58 -0700 Subject: [PATCH 01/12] fix small bug in merging --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3e59160..4382f06 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,5 +32,5 @@ services: - redis #entrypoint: ["/usr/local/bin/entrypoint.sh"] command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important -======= + From 22adc0e3619393fb1c78e8673a07f47c15c19b39 Mon Sep 17 00:00:00 2001 From: Grace Date: Tue, 24 Jun 2025 12:00:45 -0700 Subject: [PATCH 02/12] Including pycache in gitignore and fixing issue with tracking change of folder capitilization. --- src/ingest/__pycache__/__init__.cpython-310.pyc | Bin 126 -> 0 bytes src/ingest/__pycache__/bbc_rss.cpython-310.pyc | Bin 265 -> 0 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 131 -> 0 bytes .../__pycache__/celery.cpython-310.pyc | Bin 435 -> 0 bytes src/justInsight/__pycache__/tasks.cpython-310.pyc | Bin 344 -> 0 bytes 5 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/ingest/__pycache__/__init__.cpython-310.pyc delete mode 100644 src/ingest/__pycache__/bbc_rss.cpython-310.pyc delete mode 100644 src/justInsight/__pycache__/__init__.cpython-310.pyc delete mode 100644 src/justInsight/__pycache__/celery.cpython-310.pyc delete mode 100644 src/justInsight/__pycache__/tasks.cpython-310.pyc diff --git a/src/ingest/__pycache__/__init__.cpython-310.pyc b/src/ingest/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 5817e24ff69967b610e203391c0c59ca43d5ed1a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 126 zcmd1j<>g`kf~(h~GC=fW5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!HbetCXTc5y*s va;koDQL=t!UV3VAiGF;1W?p7Ve7s&kg`kf=9QbGHihKV-N=!FabFZKwQiNBvKes7+V;k7*d#m88n%bfg&gXq}CaT z!D>r@YMU7ufqH;ynf$8cJW_LV@)g4Ki*iyF72JyQa}`Q5QWb)NgB8*;b5b=KZ?P5> zW#*NvWGDie115ec=$GdgWfvDDCa3Bb7bWXw=B1|=m*^)YCC3*P7wZ*N-r^_$3WOvU rXM;3-1`#}rMJzzdPZL5TOoOnPfpWJvY;yBcN^?@}7=gSZRv-ZYv79vV diff --git a/src/justInsight/__pycache__/__init__.cpython-310.pyc b/src/justInsight/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 3dee5e1031c63744d65367d381c8f781020aeb5c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 131 zcmd1j<>g`kg7^iI86f&Gh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o2BvzdXMvySN}R zIaR;7C|N(Nw7A4GuQ)S3qeMSGJ~J<~BtBlRpz;=nO>TZlX-=vgNPjUCkYHf|0GXE? AQUCw| diff --git a/src/justInsight/__pycache__/celery.cpython-310.pyc b/src/justInsight/__pycache__/celery.cpython-310.pyc deleted file mode 100644 index 7fa8fd6bc0700002a31d0b4e94f8dd6f927a0470..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 435 zcmZWmJx{|h5VajQ{T`5*5NusCq%J}Nfe@<1#KOWvmnh90X__>N>=e}e2Yvt&11o>Y zD-(YKfxxC9Dsd-2`%cgAp6sIE?+_Eg)Aj6u67ph;&2UXIHcK5E00J~43p$~cAO!}F zNgh*Z-17+o7n;z5Hgq1Hgd&?e0f+7t@p`Z3F7c>#&oM)}nsnz&RbAvNiDy;4EfErR zBE)(>JUBWQqk3zDGOR+CD$mjGR2C^p-JFI|iaF?Jl1JGRu-;YSqRh~@OdoH}5PyH< zD~yU9RK534Y`EqJmLFSpgbc>d#LVEfrCTbRAuKc0Je0Az^cvcYzRgQ!xY`NJQgfqz zqx(|~tKgR}?SuWn+d$kEGF4?5p-?gs8&1Spa9FN%>vU1rX=mmOh}tpGql__XlT-Jd KwSUeZz0WV{Vtf7o diff --git a/src/justInsight/__pycache__/tasks.cpython-310.pyc b/src/justInsight/__pycache__/tasks.cpython-310.pyc deleted file mode 100644 index 81b5e86133e965444172b9090b65a7c75a6e43c4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 344 zcmYjLJ5Iwu5Z&3ejRi^WfM{?NUjQK_Dw;Gq{TQR6j%6&Ac`Km#zBUN%ZA5Du)e$(uNU zPHg%pitLNrqJZH7>nb?Aj1_K=Fyzr;d?Idme_BA|o*Fq$6bC^cfG)|!YihgJT?S5a z6d|QC!uM7A(uFPdm2o8oQ$Fu7KD4-5*RkY(IPdoqplW*W#sC@LPY5!ZKsY0^DAFM_ w&bzP=lC+*}+BHR-uU2Lm5CgkM=l|b7# Date: Tue, 24 Jun 2025 12:01:09 -0700 Subject: [PATCH 03/12] More capitilization changes and readme updates --- .gitignore | 10 +++++++++- readme.md | 15 ++++++++++++--- src/justInsight/celery.py | 12 +++++++++--- src/justInsight/tasks.py | 4 +++- src/justinsight/__init__.py | 0 src/justinsight/celery.py | 24 ++++++++++++++++++++++++ src/justinsight/tasks.py | 11 +++++++++++ 7 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 src/justinsight/__init__.py create mode 100644 src/justinsight/celery.py create mode 100644 src/justinsight/tasks.py diff --git a/.gitignore b/.gitignore index 96dd22a..46240d7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,10 @@ celerybeat-schedule.* -src/celerybeat-schedule.* \ No newline at end of file +src/celerybeat-schedule.* + +# Ignore all __pycache__ directories at any level +**/__pycache__/ + +# Ignore all compiled Python files +*.pyc +*.pyo +*.pyd \ No newline at end of file diff --git a/readme.md b/readme.md index c52820d..60d6d12 100644 --- a/readme.md +++ b/readme.md @@ -22,7 +22,7 @@ A lightweight, news intelligence service for ingesting free news sources, taggin cd justinsight -## Setting up Celery Beat: +## Setting up Celery Beat - On MacOS - run: docker compose up --build - OR for running in the background: docker compose up -d @@ -34,5 +34,14 @@ A lightweight, news intelligence service for ingesting free news sources, taggin - use EC2 section of entrypoint.sh - to shut down celery: docker compose down -*When running in background you can use "docker logs " to see the log and "docker ps" -to find the ID. \ No newline at end of file +# When running in background you can use "docker logs " to see the log and "docker ps" to find the ID. + +## How to Create and Schedule a Task +1) create a method for your task wherever you are working on it that can be called and complete the task. It should be in a package (ie there should be an __init__.py file in the same folder as the file you are working on). +2) Go to src/justinsight/tasks.py and create a new task following the format of the sample task. Remember to add an import statement if one is needed. +3) Go to src/justinsight/celery.py and schedule a new task after the comment showing where to schedule them. The new task can follow the same format as those above + "sample-task-name": { + "task": "justinsight.tasks.your_task_name", + "schedule": x, #where x is the number of seconds between when the task should happen + "args": (), #potential arguments for your task + }, \ No newline at end of file diff --git a/src/justInsight/celery.py b/src/justInsight/celery.py index 5a92454..b1bdfa8 100644 --- a/src/justInsight/celery.py +++ b/src/justInsight/celery.py @@ -2,17 +2,23 @@ from celery import Celery app = Celery( - "justInsight", + "justinsight", broker="redis://redis:6379/0", backend="redis://redis:6379/0", - include=["justInsight.tasks"], # adjust this to your tasks file + include=["justinsight.tasks"], ) # Optional beat schedule app.conf.beat_schedule = { "sample-task-every-10-seconds": { - "task": "justInsight.tasks.sample_task", + "task": "justinsight.tasks.sample_task", "schedule": 10.0, "args": (), }, + "sample-task-every-5-seconds": { + "task": "justinsight.tasks.sample_task", + "schedule": 5.0, + "args": (), + }, + #schedule more tasks here } diff --git a/src/justInsight/tasks.py b/src/justInsight/tasks.py index 9378f65..2b7057a 100644 --- a/src/justInsight/tasks.py +++ b/src/justInsight/tasks.py @@ -6,4 +6,6 @@ @shared_task def sample_task(): testTask() - return "done!" \ No newline at end of file + return "done!" + +#Add more tasks here in the format of the one above \ No newline at end of file diff --git a/src/justinsight/__init__.py b/src/justinsight/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/justinsight/celery.py b/src/justinsight/celery.py new file mode 100644 index 0000000..b1bdfa8 --- /dev/null +++ b/src/justinsight/celery.py @@ -0,0 +1,24 @@ +# celery.py +from celery import Celery + +app = Celery( + "justinsight", + broker="redis://redis:6379/0", + backend="redis://redis:6379/0", + include=["justinsight.tasks"], +) + +# Optional beat schedule +app.conf.beat_schedule = { + "sample-task-every-10-seconds": { + "task": "justinsight.tasks.sample_task", + "schedule": 10.0, + "args": (), + }, + "sample-task-every-5-seconds": { + "task": "justinsight.tasks.sample_task", + "schedule": 5.0, + "args": (), + }, + #schedule more tasks here +} diff --git a/src/justinsight/tasks.py b/src/justinsight/tasks.py new file mode 100644 index 0000000..2b7057a --- /dev/null +++ b/src/justinsight/tasks.py @@ -0,0 +1,11 @@ +# tasks.py +from celery import shared_task +from ingest.bbc_rss import testTask + + +@shared_task +def sample_task(): + testTask() + return "done!" + +#Add more tasks here in the format of the one above \ No newline at end of file From c497cdb7f44f3a51a295ad5a740f0495f8a0079f Mon Sep 17 00:00:00 2001 From: Grace Date: Tue, 24 Jun 2025 12:01:47 -0700 Subject: [PATCH 04/12] Editing readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 60d6d12..ca80c03 100644 --- a/readme.md +++ b/readme.md @@ -34,7 +34,7 @@ A lightweight, news intelligence service for ingesting free news sources, taggin - use EC2 section of entrypoint.sh - to shut down celery: docker compose down -# When running in background you can use "docker logs " to see the log and "docker ps" to find the ID. +### When running in background you can use "docker logs " to see the log and "docker ps" to find the ID. ## How to Create and Schedule a Task 1) create a method for your task wherever you are working on it that can be called and complete the task. It should be in a package (ie there should be an __init__.py file in the same folder as the file you are working on). From 16dfc1d0d4c74436532269a2395386cf15fa5f59 Mon Sep 17 00:00:00 2001 From: Grace Date: Tue, 24 Jun 2025 12:03:55 -0700 Subject: [PATCH 05/12] Another readme change --- readme.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index ca80c03..258d41f 100644 --- a/readme.md +++ b/readme.md @@ -34,13 +34,14 @@ A lightweight, news intelligence service for ingesting free news sources, taggin - use EC2 section of entrypoint.sh - to shut down celery: docker compose down -### When running in background you can use "docker logs " to see the log and "docker ps" to find the ID. +Note: When running in background you can use "docker logs " to see the log and "docker ps" to find the ID. ## How to Create and Schedule a Task -1) create a method for your task wherever you are working on it that can be called and complete the task. It should be in a package (ie there should be an __init__.py file in the same folder as the file you are working on). +1) Create a method for your task wherever you are working on it that can be called and complete the task. It should be in a package (ie there should be an \_\_init\_\_.py file in the same folder as the file you are working on). 2) Go to src/justinsight/tasks.py and create a new task following the format of the sample task. Remember to add an import statement if one is needed. 3) Go to src/justinsight/celery.py and schedule a new task after the comment showing where to schedule them. The new task can follow the same format as those above - "sample-task-name": { + + "sample-task-name": { "task": "justinsight.tasks.your_task_name", "schedule": x, #where x is the number of seconds between when the task should happen "args": (), #potential arguments for your task From 53631ad36e6393d7ef28a98328dcca30ac60447b Mon Sep 17 00:00:00 2001 From: Grace Date: Tue, 24 Jun 2025 12:04:55 -0700 Subject: [PATCH 06/12] More formatting --- readme.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index 258d41f..fa61a1f 100644 --- a/readme.md +++ b/readme.md @@ -41,8 +41,8 @@ Note: When running in background you can use "docker logs 2) Go to src/justinsight/tasks.py and create a new task following the format of the sample task. Remember to add an import statement if one is needed. 3) Go to src/justinsight/celery.py and schedule a new task after the comment showing where to schedule them. The new task can follow the same format as those above - "sample-task-name": { - "task": "justinsight.tasks.your_task_name", - "schedule": x, #where x is the number of seconds between when the task should happen - "args": (), #potential arguments for your task + "sample-task-name": { + "task": "justinsight.tasks.your_task_name", + "schedule": x, #where x is the number of seconds between when the task should happen + "args": (), #potential arguments for your task }, \ No newline at end of file From 923d06f3e897502217b81a216b89182954481482 Mon Sep 17 00:00:00 2001 From: Grace Date: Tue, 24 Jun 2025 12:07:45 -0700 Subject: [PATCH 07/12] Remove old version of the folder --- src/justInsight/__init__.py | 0 src/justInsight/celery.py | 24 ------------------------ src/justInsight/tasks.py | 11 ----------- 3 files changed, 35 deletions(-) delete mode 100644 src/justInsight/__init__.py delete mode 100644 src/justInsight/celery.py delete mode 100644 src/justInsight/tasks.py diff --git a/src/justInsight/__init__.py b/src/justInsight/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/justInsight/celery.py b/src/justInsight/celery.py deleted file mode 100644 index b1bdfa8..0000000 --- a/src/justInsight/celery.py +++ /dev/null @@ -1,24 +0,0 @@ -# celery.py -from celery import Celery - -app = Celery( - "justinsight", - broker="redis://redis:6379/0", - backend="redis://redis:6379/0", - include=["justinsight.tasks"], -) - -# Optional beat schedule -app.conf.beat_schedule = { - "sample-task-every-10-seconds": { - "task": "justinsight.tasks.sample_task", - "schedule": 10.0, - "args": (), - }, - "sample-task-every-5-seconds": { - "task": "justinsight.tasks.sample_task", - "schedule": 5.0, - "args": (), - }, - #schedule more tasks here -} diff --git a/src/justInsight/tasks.py b/src/justInsight/tasks.py deleted file mode 100644 index 2b7057a..0000000 --- a/src/justInsight/tasks.py +++ /dev/null @@ -1,11 +0,0 @@ -# tasks.py -from celery import shared_task -from ingest.bbc_rss import testTask - - -@shared_task -def sample_task(): - testTask() - return "done!" - -#Add more tasks here in the format of the one above \ No newline at end of file From cec7aeabb82bf1d2ae9489bc2e274235c564662e Mon Sep 17 00:00:00 2001 From: Grace Date: Wed, 25 Jun 2025 09:50:33 -0700 Subject: [PATCH 08/12] Fixing weird bug in docker-compose --- bbc_world.xml | 242 -------------------------------------- docker-compose.yml | 11 +- requirements.txt | 5 +- src/justinsight/celery.py | 8 +- src/justinsight/tasks.py | 4 +- 5 files changed, 18 insertions(+), 252 deletions(-) delete mode 100644 bbc_world.xml diff --git a/bbc_world.xml b/bbc_world.xml deleted file mode 100644 index 35a7e7e..0000000 --- a/bbc_world.xml +++ /dev/null @@ -1,242 +0,0 @@ - - - <![CDATA[BBC News]]> - - https://www.bbc.co.uk/news/world - - https://news.bbcimg.co.uk/nol/shared/img/bbc_news_120x60.gif - BBC News - https://www.bbc.co.uk/news/world - - RSS for Node - Mon, 23 Jun 2025 16:59:28 GMT - - - - 15 - - <![CDATA[Israel says it struck Tehran's Evin prison and Fordo access routes]]> - - https://www.bbc.com/news/articles/cp8621gnknjo - https://www.bbc.com/news/articles/cp8621gnknjo#0 - Mon, 23 Jun 2025 16:29:51 GMT - - - - <![CDATA['Why wouldn't there be a regime change?' Trump's latest on Iran]]> - - https://www.bbc.com/news/articles/cp8m3861637o - https://www.bbc.com/news/articles/cp8m3861637o#0 - Mon, 23 Jun 2025 13:01:31 GMT - - - - <![CDATA[Bride shot dead in attack on French wedding party]]> - - https://www.bbc.com/news/articles/cn0q8pypxwxo - https://www.bbc.com/news/articles/cn0q8pypxwxo#0 - Mon, 23 Jun 2025 00:50:23 GMT - - - - <![CDATA[Nine killed in Kyiv in intense Russian air attack]]> - - https://www.bbc.com/news/articles/c0k7vll16l3o - https://www.bbc.com/news/articles/c0k7vll16l3o#0 - Mon, 23 Jun 2025 13:12:01 GMT - - - - <![CDATA[Scottish wingsuit flyer Liam Byrne dies during Swiss Alps jump]]> - - https://www.bbc.com/news/articles/cwyepy2zz11o - https://www.bbc.com/news/articles/cwyepy2zz11o#0 - Mon, 23 Jun 2025 13:36:18 GMT - - - - <![CDATA[Rescuers search for Brazilian tourist on Indonesian volcano]]> - - https://www.bbc.com/news/articles/crk60k8gp8jo - https://www.bbc.com/news/articles/crk60k8gp8jo#0 - Mon, 23 Jun 2025 11:10:36 GMT - - - - <![CDATA[First celestial image unveiled from revolutionary telescope]]> - - https://www.bbc.com/news/articles/cj3rmjjgx6xo - https://www.bbc.com/news/articles/cj3rmjjgx6xo#1 - Mon, 23 Jun 2025 16:35:11 GMT - - - - <![CDATA[Nato summit could be the most significant since end of Cold War]]> - - https://www.bbc.com/news/articles/cqjqvr75v1jo - https://www.bbc.com/news/articles/cqjqvr75v1jo#1 - Mon, 23 Jun 2025 14:02:57 GMT - - - - <![CDATA[Police officers charged with murder of Kenyan blogger]]> - - https://www.bbc.com/news/articles/cz09l4k4184o - https://www.bbc.com/news/articles/cz09l4k4184o#1 - Mon, 23 Jun 2025 15:46:31 GMT - - - - <![CDATA[Dozens of soldiers seized by locals in Colombian mountains]]> - - https://www.bbc.com/news/articles/c4gdlrk1pzko - https://www.bbc.com/news/articles/c4gdlrk1pzko#1 - Mon, 23 Jun 2025 04:09:05 GMT - - - - <![CDATA[Belarus opposition leader's husband urges Trump to help other prisoners ]]> - - https://www.bbc.com/news/articles/c07dgmymd1eo - https://www.bbc.com/news/articles/c07dgmymd1eo#1 - Sun, 22 Jun 2025 17:04:56 GMT - - - - <![CDATA[Suicide bombing at Damascus church kills 25, Syrian authorities say]]> - - https://www.bbc.com/news/articles/c307n9p43z9o - https://www.bbc.com/news/articles/c307n9p43z9o#1 - Sun, 22 Jun 2025 21:26:21 GMT - - - - <![CDATA[Lyse Doucet: What the US and Iran do next could be momentous]]> - - https://www.bbc.com/news/articles/cy0w94yj68xo - https://www.bbc.com/news/articles/cy0w94yj68xo#2 - Sun, 22 Jun 2025 21:41:06 GMT - - - - <![CDATA[Decoy flights and seven B-2 stealth bombers - how US says it hit Iran's nuclear sites]]> - - https://www.bbc.com/news/articles/cew0x7159edo - https://www.bbc.com/news/articles/cew0x7159edo#2 - Sun, 22 Jun 2025 21:45:53 GMT - - - - <![CDATA[How China made electric vehicles mainstream]]> - - https://www.bbc.com/news/articles/c2d5ld8y8pwo - https://www.bbc.com/news/articles/c2d5ld8y8pwo#2 - Sun, 22 Jun 2025 23:17:26 GMT - - - - <![CDATA[Caught in the crossfire - the victims of Cape Town's gang warfare]]> - - https://www.bbc.com/news/articles/c1mg2xg339jo - https://www.bbc.com/news/articles/c1mg2xg339jo#2 - Sun, 22 Jun 2025 23:57:19 GMT - - - - <![CDATA['Tourists are our lifeline': Picking up the pieces in Kashmir after April killings]]> - - https://www.bbc.com/news/articles/cly8lx75d6ro - https://www.bbc.com/news/articles/cly8lx75d6ro#2 - Sun, 22 Jun 2025 23:53:10 GMT - - - - <![CDATA[What we know about US strikes on three Iranian nuclear sites]]> - - https://www.bbc.com/news/articles/cvg9r4q99g4o - https://www.bbc.com/news/articles/cvg9r4q99g4o#2 - Mon, 23 Jun 2025 14:29:31 GMT - - - - <![CDATA['Paralysed and constantly caught by surprise': Voices from Iran after US strikes]]> - - https://www.bbc.com/news/articles/cpwq2vnd827o - https://www.bbc.com/news/articles/cpwq2vnd827o#2 - Sun, 22 Jun 2025 16:25:21 GMT - - - - <![CDATA[How a self-styled knight giving away cars and wads of cash got people talking]]> - - https://www.bbc.com/news/articles/cn81ynyjy8no - https://www.bbc.com/news/articles/cn81ynyjy8no#2 - Sun, 22 Jun 2025 01:31:32 GMT - - - - <![CDATA[Watch: Video shows aftermath of Israeli strikes on Iran]]> - - https://www.bbc.com/news/videos/c8xvqje77ygo - https://www.bbc.com/news/videos/c8xvqje77ygo#3 - Mon, 23 Jun 2025 16:44:46 GMT - - - - <![CDATA[Netanyahu and Trump 'totally aligned' on Iran - Barkat]]> - - https://www.bbc.com/news/videos/c3en18eq435o - https://www.bbc.com/news/videos/c3en18eq435o#3 - Mon, 23 Jun 2025 15:31:40 GMT - - - - <![CDATA[Watch: How successful have the US strikes on Iran been?]]> - - https://www.bbc.com/news/videos/cq53l9dvggjo - https://www.bbc.com/news/videos/cq53l9dvggjo#3 - Sun, 22 Jun 2025 20:39:06 GMT - - - - <![CDATA[US strikes on Iran trigger protests internationally]]> - - https://www.bbc.com/news/videos/c4g2wxwel5qo - https://www.bbc.com/news/videos/c4g2wxwel5qo#3 - Mon, 23 Jun 2025 01:46:57 GMT - - - - <![CDATA[Watch: Trump says strikes on Iran a 'spectacular military success' in address]]> - - https://www.bbc.com/news/videos/c93k8nvdnqpo - https://www.bbc.com/news/videos/c93k8nvdnqpo#3 - Sun, 22 Jun 2025 02:52:36 GMT - - - - <![CDATA['Necessary' or 'unfounded'? Americans in Texas respond to US strikes]]> - - https://www.bbc.com/news/videos/c07dg04ke45o - https://www.bbc.com/news/videos/c07dg04ke45o#3 - Sun, 22 Jun 2025 22:47:25 GMT - - - - <![CDATA[Satellite images and decoy planes: Behind the US attacks on Iran]]> - - https://www.bbc.com/news/videos/cdezkx5nl1wo - https://www.bbc.com/news/videos/cdezkx5nl1wo#3 - Sun, 22 Jun 2025 18:02:12 GMT - - - - <![CDATA[Watch: The US has joined the Iran-Israel war. What happens now?]]> - - https://www.bbc.com/news/videos/c5ypw09gdzpo - https://www.bbc.com/news/videos/c5ypw09gdzpo#3 - Sun, 22 Jun 2025 13:58:09 GMT - - - - \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4382f06..f3635ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,4 +33,13 @@ services: #entrypoint: ["/usr/local/bin/entrypoint.sh"] command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important - + celery_beat: + build: + context: . + dockerfile: .docker/Dockerfile + volumes: + - .:/workspace:cached + depends_on: + - redis + #entrypoint: ["/usr/local/bin/entrypoint.sh"] + command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1ce978e..e7db3d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,4 @@ -celery[redis] \ No newline at end of file +celery[redis] +feedparser +requests +beautifulsoup4 \ No newline at end of file diff --git a/src/justinsight/celery.py b/src/justinsight/celery.py index b1bdfa8..e014e33 100644 --- a/src/justinsight/celery.py +++ b/src/justinsight/celery.py @@ -10,15 +10,11 @@ # Optional beat schedule app.conf.beat_schedule = { - "sample-task-every-10-seconds": { - "task": "justinsight.tasks.sample_task", - "schedule": 10.0, - "args": (), - }, - "sample-task-every-5-seconds": { + "sample-task-every-5-seconds": { "task": "justinsight.tasks.sample_task", "schedule": 5.0, "args": (), }, + #schedule more tasks here } diff --git a/src/justinsight/tasks.py b/src/justinsight/tasks.py index 2b7057a..56425e7 100644 --- a/src/justinsight/tasks.py +++ b/src/justinsight/tasks.py @@ -1,11 +1,11 @@ # tasks.py from celery import shared_task -from ingest.bbc_rss import testTask +#from ingest.bbc_rss import testTask @shared_task def sample_task(): - testTask() + print("Sample task!") return "done!" #Add more tasks here in the format of the one above \ No newline at end of file From e8e62e73d1ffdc46fd140c83179fb5eaa78683cf Mon Sep 17 00:00:00 2001 From: Grace Date: Wed, 25 Jun 2025 10:18:14 -0700 Subject: [PATCH 09/12] Tasks set up for BBC and NYT --- .gitignore | 3 +++ requirements.txt | 3 ++- src/justinsight/celery.py | 18 +++++++++++++++--- src/justinsight/tasks.py | 13 ++++++++++++- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 4af9dfe..3742713 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ feed*.json *.pyc *.pyo *.pyd + +#Ignore saved data +data/* \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e7db3d3..9fca38f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ celery[redis] feedparser requests -beautifulsoup4 \ No newline at end of file +beautifulsoup4 +playwright \ No newline at end of file diff --git a/src/justinsight/celery.py b/src/justinsight/celery.py index e014e33..0292a0b 100644 --- a/src/justinsight/celery.py +++ b/src/justinsight/celery.py @@ -10,9 +10,21 @@ # Optional beat schedule app.conf.beat_schedule = { - "sample-task-every-5-seconds": { - "task": "justinsight.tasks.sample_task", - "schedule": 5.0, + # "sample-task-every-5-seconds": { + # "task": "justinsight.tasks.sample_task", + # "schedule": 5.0, + # "args": (), + # }, + + "check-BBCfeed-every-5-minutes": { + "task": "justinsight.tasks.bbcLogger_task", + "schedule": 300.0, + "args": (), + }, + + "check-NYTfeed-every-5-minutes": { + "task": "justinsight.tasks.nytLogger_task", + "schedule": 300.0, "args": (), }, diff --git a/src/justinsight/tasks.py b/src/justinsight/tasks.py index 56425e7..36b6c33 100644 --- a/src/justinsight/tasks.py +++ b/src/justinsight/tasks.py @@ -1,6 +1,7 @@ # tasks.py from celery import shared_task -#from ingest.bbc_rss import testTask +from ingest.bbc_rss import check_and_save_new_entries as check_and_save_bbc +from ingest.nyt_rss import check_and_save_new_entries as check_and_save_nyt @shared_task @@ -8,4 +9,14 @@ def sample_task(): print("Sample task!") return "done!" +@shared_task +def bbcLogger_task(): + check_and_save_bbc() + return "BBC RSS Feed checked." + +@shared_task +def nytLogger_task(): + check_and_save_nyt() + return "NYT RSS Feed checked." + #Add more tasks here in the format of the one above \ No newline at end of file From 5ed3d0123bbf25b82563ea4f9b90d5560a1e1603 Mon Sep 17 00:00:00 2001 From: Grace Date: Thu, 26 Jun 2025 09:32:10 -0700 Subject: [PATCH 10/12] Updating docker file to include mongo --- docker-compose.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f3635ce..1fe54f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,7 @@ services: dockerfile: .docker/Dockerfile volumes: - .:/workspace:cached + - mongo_data:/data/db depends_on: - redis #entrypoint: ["/usr/local/bin/entrypoint.sh"] @@ -42,4 +43,20 @@ services: depends_on: - redis #entrypoint: ["/usr/local/bin/entrypoint.sh"] - command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important \ No newline at end of file + command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important + + mongo: + image: mongo:latest + container_name: mongo + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: myuser + MONGO_INITDB_ROOT_PASSWORD: mypassword + ports: + - "27017:27017" + volumes: + - mongo_data:/data/db + - .:/workspace:cached + +volumes: + mongo_data: \ No newline at end of file From 820c6f31e17e697577a8eaf684131e81d786f999 Mon Sep 17 00:00:00 2001 From: Grace Date: Thu, 26 Jun 2025 09:53:20 -0700 Subject: [PATCH 11/12] Adding requirements --- requirements.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9fca38f..18115a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ +beautifulsoup4 celery[redis] feedparser -requests -beautifulsoup4 -playwright \ No newline at end of file +playwright +pymongo +requests \ No newline at end of file From 7a2b1ebcf581972dafc86d3ec53e63cf4e7f970f Mon Sep 17 00:00:00 2001 From: Grace Date: Thu, 26 Jun 2025 12:16:50 -0700 Subject: [PATCH 12/12] mongodb up and running now, class structure for rss feed readers implemented for bbc. --- docker-compose.yml | 1 + src/ingest/base_ingestor.py | 52 ++++++++++++++++++++++++++++++++++ src/ingest/bbc_ingestor.py | 15 ++++++++++ src/ingest/save_to_database.py | 18 ++++++++++++ src/justinsight/celery.py | 12 ++++---- src/justinsight/tasks.py | 7 +++-- 6 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 src/ingest/base_ingestor.py create mode 100644 src/ingest/bbc_ingestor.py create mode 100644 src/ingest/save_to_database.py diff --git a/docker-compose.yml b/docker-compose.yml index 1fe54f8..8d43ec9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ services: - mongo_data:/data/db depends_on: - redis + - mongo #entrypoint: ["/usr/local/bin/entrypoint.sh"] command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important diff --git a/src/ingest/base_ingestor.py b/src/ingest/base_ingestor.py new file mode 100644 index 0000000..0c5eaee --- /dev/null +++ b/src/ingest/base_ingestor.py @@ -0,0 +1,52 @@ +import datetime +import feedparser +import hashlib +import re +from src.ingest.save_to_database import save_entry + +class BaseIngestor: + RSS_URL = None #will be set by the subclasses + + def slugify(self, text): + # Convert title to a filesystem-friendly slug + text = text.lower() + text = re.sub(r'[^a-z0-9]+', '-', text) + return text.strip('-') + + def format_date(self, entry): + # Extract and format the published date + try: + dt = datetime.datetime(*entry.published_parsed[:6]) + return dt.strftime("%Y-%m-%d") + except: + return "unknown-date" + + def fetch_full_text(self, url): + raise NotImplementedError( + "Subclass must implement fetch_full_text()" + ) + + def generate_entry_hash(self, entry): + hash_input = f"{entry.title}{entry.link}{entry.get('published', '')}" + return hashlib.sha256(hash_input.encode('utf-8')).hexdigest() + + def format_entry(self, entry): + title_slug = self.slugify(entry.title) + date_str = self.format_date(entry) + full_text = self.fetch_full_text(entry.link) + + data = { + "title": entry.title, + "link": entry.link, + "published": entry.get("published", ""), + "summary": entry.get("summary", ""), + "full_text": full_text, + "id": self.generate_entry_hash(entry) + } + return data + + def check_and_save_new_entries(self): + feed = feedparser.parse(self.RSS_URL) + + for entry in feed.entries: + save_entry(self.format_entry(entry)) diff --git a/src/ingest/bbc_ingestor.py b/src/ingest/bbc_ingestor.py new file mode 100644 index 0000000..71d5542 --- /dev/null +++ b/src/ingest/bbc_ingestor.py @@ -0,0 +1,15 @@ +from bs4 import BeautifulSoup +import requests +from src.ingest.base_ingestor import BaseIngestor + +class BBCIngestor(BaseIngestor): + RSS_URL = "http://feeds.bbci.co.uk/news/world/rss.xml" + + def fetch_full_text(self, article_url): + response = requests.get(self.RSS_URL) + soup = BeautifulSoup(response.content, 'html.parser') + + article = soup.find('article') + + if article: + return(article.get_text()) \ No newline at end of file diff --git a/src/ingest/save_to_database.py b/src/ingest/save_to_database.py new file mode 100644 index 0000000..3feb6f7 --- /dev/null +++ b/src/ingest/save_to_database.py @@ -0,0 +1,18 @@ +from pymongo import MongoClient + +# Include username, password, and authentication database +client = MongoClient("mongodb://myuser:mypassword@mongo:27017/justinsightdb?authSource=admin") + +# Get (or create) a database +db = client["justinsightdb"] + +# Get (or create) a collection +collection = db["articles"] + +def save_entry(entry): + #check if the entry has already been saved and if it has not then save it + entry_hash = entry["id"] + if collection.count_documents({"id": entry_hash}) == 0: + collection.insert_one(entry) + else: + print(f"{entry['title']} already in use!") diff --git a/src/justinsight/celery.py b/src/justinsight/celery.py index 0292a0b..b5b241a 100644 --- a/src/justinsight/celery.py +++ b/src/justinsight/celery.py @@ -18,15 +18,15 @@ "check-BBCfeed-every-5-minutes": { "task": "justinsight.tasks.bbcLogger_task", - "schedule": 300.0, + "schedule": 5.0, "args": (), }, - "check-NYTfeed-every-5-minutes": { - "task": "justinsight.tasks.nytLogger_task", - "schedule": 300.0, - "args": (), - }, + # "check-NYTfeed-every-5-minutes": { + # "task": "justinsight.tasks.nytLogger_task", + # "schedule": 300.0, + # "args": (), + # }, #schedule more tasks here } diff --git a/src/justinsight/tasks.py b/src/justinsight/tasks.py index 36b6c33..7066141 100644 --- a/src/justinsight/tasks.py +++ b/src/justinsight/tasks.py @@ -1,7 +1,6 @@ # tasks.py from celery import shared_task -from ingest.bbc_rss import check_and_save_new_entries as check_and_save_bbc -from ingest.nyt_rss import check_and_save_new_entries as check_and_save_nyt +from ingest.bbc_ingestor import BBCIngestor @shared_task @@ -11,7 +10,9 @@ def sample_task(): @shared_task def bbcLogger_task(): - check_and_save_bbc() + # Create an instance of the class + ingestor = BBCIngestor() + ingestor.check_and_save_new_entries() # this will invoke the inherited logic return "BBC RSS Feed checked." @shared_task