Ingestion Scheduler with Celery and Docker

.docker/Dockerfile

@@ -6,15 +6,23 @@ RUN apt-get update && \
  apt-get install -y python3 python3-pip git && \
  rm -rf /var/lib/apt/lists/*
 
+# Install requirements
+COPY requirements.txt /workspace/requirements.txt
+RUN pip install --no-cache-dir -r /workspace/requirements.txt
+
 # Set working directory
 WORKDIR /workspace
 
+# Ensure Python can import from this path
+#ENV PYTHONPATH=/workspace/src
+ENV PYTHONPATH=/workspace:/workspace/src
+
 
 # Copy the entrypoint from the .docker folder into the image
 COPY .docker/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
 
-ENTRYPOINT ["/use/local/bin/entrypoint.sh"]
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 CMD ["bash"]
 
 

.docker/entrypoint.sh

@@ -1,32 +1,47 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Default branch (override via env if you want)
-BRANCH="${BRANCH:-master}"
-
-# Tell Git that /workspace is safe to operate on
-git config --global --add safe.directory /workspace
-
-# Bail early if we don’t see a .git folder
-if [ ! -d .git ]; then
- echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
- exit 1
-fi
-
-# Update & switch branches
-echo "⏳ Fetching updates..."
-git fetch origin --prune
-
-echo "⏳ Checking out branch '$BRANCH'..."
-# if branch already exists locally, switch; otherwise track remote
-if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
- git checkout "$BRANCH"
-else
- git checkout --track "origin/$BRANCH"
-fi
-
-echo "⏳ Pulling latest changes..."
-git pull --ff-only origin "$BRANCH"
+#use this code for deployment on EC2
+#######################################################################################
+# # Default branch (override via env if you want)
+# BRANCH="${BRANCH:-master}"
+
+# # Tell Git that /workspace is safe to operate on
+# git config --global --add safe.directory /workspace
+
+# # Bail early if we don’t see a .git folder
+# if [ ! -d .git ]; then
+# echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
+# exit 1
+# fi
+
+# # Update & switch branches
+# echo "⏳ Fetching updates..."
+# git fetch origin --prune
+
+# echo "⏳ Checking out branch '$BRANCH'..."
+# # if branch already exists locally, switch; otherwise track remote
+# if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
+# git checkout "$BRANCH"
+# else
+# git checkout --track "origin/$BRANCH"
+# fi
+
+# echo "⏳ Pulling latest changes..."
+# git pull --ff-only origin "$BRANCH"
+#######################################################################################
+
+
+#use this code for local development
+#######################################################################################
+# Skip Git checkout for local dev — just use the mounted code as-is
+echo "Skipping Git checkout — using mounted code in /workspace"
+
+# Set the Python path and working dir
+#cd /workspace/src
+#export PYTHONPATH=/workspace/src
+#######################################################################################
+
 
 # Hand off to whatever was passed in (e.g. bash)
 exec "${@:-bash}"

.dockerignore

@@ -0,0 +1,4 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*
+.github
+.gitignore

.gitignore

@@ -0,0 +1,2 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*

docker-compose.gpu.yml

@@ -0,0 +1,8 @@
+version: "3.8"
+services:
+ justinsight_service:
+ gpus: all
+ celery_worker:
+ gpus: all
+ celery_beat:
+ gpus: all

docker-compose.yml

@@ -1,25 +1,46 @@
 version: '3.8'
-
+
 services:
  justinsight_service:
  build:
  context: .
  dockerfile: .docker/Dockerfile
- gpus: all
 
  # bind-mount your repo and the shared EBS volume
  volumes:
  - ./:/workspace:cached
  - ./shared:/workspace/shared:cached
 
  # ensure entrypoint.sh runs (clones & checks out the right branch)
- entrypoint: ["/usr/local/bin/entrypoint.sh"]
+#entrypoint: ["/usr/local/bin/entrypoint.sh"]
  command: ["bash"]
  stdin_open: true
  tty: true
 
 
- # optional: default branch (can be overridden per-run)
- environment:
- - BRANCH=${BRANCH:-master}
+ redis:
+ image: redis:7-alpine
+ ports:
+ - "6379:6379"
+
+ celery_worker:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important
 
+ celery_beat:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important

readme.md

@@ -19,4 +19,20 @@ A lightweight, news intelligence service for ingesting free news sources, taggin
 1. **Clone** the repo:
  ```bash
  git clone https://github.com/JustInternetAI/JustInsight.git
- cd justinsight
+ cd justinsight
+
+
+## Setting up Celery Beat:
+- On MacOS
+ - run: docker compose up --build
+ - OR for running in the background: docker compose up -d
+ - use local development section of entrypoint.sh
+ - to shut down celery: docker compose down
+- On the EC2
+ - run: docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build
+ - OR for running in the background: docker compose up -d
+ - use EC2 section of entrypoint.sh
+ - to shut down celery: docker compose down
+
+*When running in background you can use "docker logs <container_name_or_id>" to see the log and "docker ps"
+to find the ID.

requirements.txt

@@ -0,0 +1 @@
+celery[redis]

src/ingest/bbc_rss.py

@@ -1 +1,3 @@
-print("Hello World!")
+
+def testTask():
+ print("Hello World! From the RSS file")

src/justInsight/celery.py

@@ -0,0 +1,18 @@
+# celery.py
+from celery import Celery
+
+app = Celery(
+ "justInsight",
+ broker="redis://redis:6379/0",
+ backend="redis://redis:6379/0",
+ include=["justInsight.tasks"], # adjust this to your tasks file
+)
+
+# Optional beat schedule
+app.conf.beat_schedule = {
+ "sample-task-every-10-seconds": {
+ "task": "justInsight.tasks.sample_task",
+ "schedule": 10.0,
+ "args": (),
+ },
+}

src/justInsight/tasks.py

@@ -0,0 +1,9 @@
+# tasks.py
+from celery import shared_task
+from ingest.bbc_rss import testTask
+
+
+@shared_task
+def sample_task():
+ testTask()
+ return "done!"

.docker/Dockerfile

@@ -6,15 +6,23 @@ RUN apt-get update && \
  apt-get install -y python3 python3-pip git && \
  rm -rf /var/lib/apt/lists/*
 
+# Install requirements
+COPY requirements.txt /workspace/requirements.txt
+RUN pip install --no-cache-dir -r /workspace/requirements.txt
+
 # Set working directory
 WORKDIR /workspace
 
+# Ensure Python can import from this path
+#ENV PYTHONPATH=/workspace/src
+ENV PYTHONPATH=/workspace:/workspace/src
+
 
 # Copy the entrypoint from the .docker folder into the image
 COPY .docker/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
 
-ENTRYPOINT ["/use/local/bin/entrypoint.sh"]
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 CMD ["bash"]
 
 

.docker/entrypoint.sh

@@ -1,32 +1,47 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Default branch (override via env if you want)
-BRANCH="${BRANCH:-master}"
-
-# Tell Git that /workspace is safe to operate on
-git config --global --add safe.directory /workspace
-
-# Bail early if we don’t see a .git folder
-if [ ! -d .git ]; then
- echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
- exit 1
-fi
-
-# Update & switch branches
-echo "⏳ Fetching updates..."
-git fetch origin --prune
-
-echo "⏳ Checking out branch '$BRANCH'..."
-# if branch already exists locally, switch; otherwise track remote
-if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
- git checkout "$BRANCH"
-else
- git checkout --track "origin/$BRANCH"
-fi
-
-echo "⏳ Pulling latest changes..."
-git pull --ff-only origin "$BRANCH"
+#use this code for deployment on EC2
+#######################################################################################
+# # Default branch (override via env if you want)
+# BRANCH="${BRANCH:-master}"
+
+# # Tell Git that /workspace is safe to operate on
+# git config --global --add safe.directory /workspace
+
+# # Bail early if we don’t see a .git folder
+# if [ ! -d .git ]; then
+# echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
+# exit 1
+# fi
+
+# # Update & switch branches
+# echo "⏳ Fetching updates..."
+# git fetch origin --prune
+
+# echo "⏳ Checking out branch '$BRANCH'..."
+# # if branch already exists locally, switch; otherwise track remote
+# if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
+# git checkout "$BRANCH"
+# else
+# git checkout --track "origin/$BRANCH"
+# fi
+
+# echo "⏳ Pulling latest changes..."
+# git pull --ff-only origin "$BRANCH"
+#######################################################################################
+
+
+#use this code for local development
+#######################################################################################
+# Skip Git checkout for local dev — just use the mounted code as-is
+echo "Skipping Git checkout — using mounted code in /workspace"
+
+# Set the Python path and working dir
+#cd /workspace/src
+#export PYTHONPATH=/workspace/src
+#######################################################################################
+
 
 # Hand off to whatever was passed in (e.g. bash)
 exec "${@:-bash}"

.dockerignore

@@ -0,0 +1,4 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*
+.github
+.gitignore

.gitignore

@@ -0,0 +1,2 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*

docker-compose.gpu.yml

@@ -0,0 +1,8 @@
+version: "3.8"
+services:
+ justinsight_service:
+ gpus: all
+ celery_worker:
+ gpus: all
+ celery_beat:
+ gpus: all

docker-compose.yml

@@ -1,25 +1,46 @@
 version: '3.8'
-
+
 services:
  justinsight_service:
  build:
  context: .
  dockerfile: .docker/Dockerfile
- gpus: all
 
  # bind-mount your repo and the shared EBS volume
  volumes:
  - ./:/workspace:cached
  - ./shared:/workspace/shared:cached
 
  # ensure entrypoint.sh runs (clones & checks out the right branch)
- entrypoint: ["/usr/local/bin/entrypoint.sh"]
+#entrypoint: ["/usr/local/bin/entrypoint.sh"]
  command: ["bash"]
  stdin_open: true
  tty: true
 
 
- # optional: default branch (can be overridden per-run)
- environment:
- - BRANCH=${BRANCH:-master}
+ redis:
+ image: redis:7-alpine
+ ports:
+ - "6379:6379"
+
+ celery_worker:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important
 
+ celery_beat:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important

readme.md

@@ -19,4 +19,20 @@ A lightweight, news intelligence service for ingesting free news sources, taggin
 1. **Clone** the repo:
  ```bash
  git clone https://github.com/JustInternetAI/JustInsight.git
- cd justinsight
+ cd justinsight
+
+
+## Setting up Celery Beat:
+- On MacOS
+ - run: docker compose up --build
+ - OR for running in the background: docker compose up -d
+ - use local development section of entrypoint.sh
+ - to shut down celery: docker compose down
+- On the EC2
+ - run: docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build
+ - OR for running in the background: docker compose up -d
+ - use EC2 section of entrypoint.sh
+ - to shut down celery: docker compose down
+
+*When running in background you can use "docker logs <container_name_or_id>" to see the log and "docker ps"
+to find the ID.

requirements.txt

@@ -0,0 +1 @@
+celery[redis]

src/ingest/bbc_rss.py

@@ -1 +1,3 @@
-print("Hello World!")
+
+def testTask():
+ print("Hello World! From the RSS file")

src/justInsight/celery.py

@@ -0,0 +1,18 @@
+# celery.py
+from celery import Celery
+
+app = Celery(
+ "justInsight",
+ broker="redis://redis:6379/0",
+ backend="redis://redis:6379/0",
+ include=["justInsight.tasks"], # adjust this to your tasks file
+)
+
+# Optional beat schedule
+app.conf.beat_schedule = {
+ "sample-task-every-10-seconds": {
+ "task": "justInsight.tasks.sample_task",
+ "schedule": 10.0,
+ "args": (),
+ },
+}

src/justInsight/tasks.py

@@ -0,0 +1,9 @@
+# tasks.py
+from celery import shared_task
+from ingest.bbc_rss import testTask
+
+
+@shared_task
+def sample_task():
+ testTask()
+ return "done!"

.docker/Dockerfile

@@ -6,15 +6,23 @@ RUN apt-get update && \
  apt-get install -y python3 python3-pip git && \
  rm -rf /var/lib/apt/lists/*
 
+# Install requirements
+COPY requirements.txt /workspace/requirements.txt
+RUN pip install --no-cache-dir -r /workspace/requirements.txt
+
 # Set working directory
 WORKDIR /workspace
 
+# Ensure Python can import from this path
+#ENV PYTHONPATH=/workspace/src
+ENV PYTHONPATH=/workspace:/workspace/src
+
 
 # Copy the entrypoint from the .docker folder into the image
 COPY .docker/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
 
-ENTRYPOINT ["/use/local/bin/entrypoint.sh"]
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 CMD ["bash"]
 
 

.docker/entrypoint.sh

@@ -1,32 +1,47 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Default branch (override via env if you want)
-BRANCH="${BRANCH:-master}"
-
-# Tell Git that /workspace is safe to operate on
-git config --global --add safe.directory /workspace
-
-# Bail early if we don’t see a .git folder
-if [ ! -d .git ]; then
- echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
- exit 1
-fi
-
-# Update & switch branches
-echo "⏳ Fetching updates..."
-git fetch origin --prune
-
-echo "⏳ Checking out branch '$BRANCH'..."
-# if branch already exists locally, switch; otherwise track remote
-if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
- git checkout "$BRANCH"
-else
- git checkout --track "origin/$BRANCH"
-fi
-
-echo "⏳ Pulling latest changes..."
-git pull --ff-only origin "$BRANCH"
+#use this code for deployment on EC2
+#######################################################################################
+# # Default branch (override via env if you want)
+# BRANCH="${BRANCH:-master}"
+
+# # Tell Git that /workspace is safe to operate on
+# git config --global --add safe.directory /workspace
+
+# # Bail early if we don’t see a .git folder
+# if [ ! -d .git ]; then
+# echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
+# exit 1
+# fi
+
+# # Update & switch branches
+# echo "⏳ Fetching updates..."
+# git fetch origin --prune
+
+# echo "⏳ Checking out branch '$BRANCH'..."
+# # if branch already exists locally, switch; otherwise track remote
+# if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
+# git checkout "$BRANCH"
+# else
+# git checkout --track "origin/$BRANCH"
+# fi
+
+# echo "⏳ Pulling latest changes..."
+# git pull --ff-only origin "$BRANCH"
+#######################################################################################
+
+
+#use this code for local development
+#######################################################################################
+# Skip Git checkout for local dev — just use the mounted code as-is
+echo "Skipping Git checkout — using mounted code in /workspace"
+
+# Set the Python path and working dir
+#cd /workspace/src
+#export PYTHONPATH=/workspace/src
+#######################################################################################
+
 
 # Hand off to whatever was passed in (e.g. bash)
 exec "${@:-bash}"

.dockerignore

@@ -0,0 +1,4 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*
+.github
+.gitignore

.gitignore

@@ -0,0 +1,2 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*

docker-compose.gpu.yml

@@ -0,0 +1,8 @@
+version: "3.8"
+services:
+ justinsight_service:
+ gpus: all
+ celery_worker:
+ gpus: all
+ celery_beat:
+ gpus: all

docker-compose.yml

@@ -1,25 +1,46 @@
 version: '3.8'
-
+
 services:
  justinsight_service:
  build:
  context: .
  dockerfile: .docker/Dockerfile
- gpus: all
 
  # bind-mount your repo and the shared EBS volume
  volumes:
  - ./:/workspace:cached
  - ./shared:/workspace/shared:cached
 
  # ensure entrypoint.sh runs (clones & checks out the right branch)
- entrypoint: ["/usr/local/bin/entrypoint.sh"]
+#entrypoint: ["/usr/local/bin/entrypoint.sh"]
  command: ["bash"]
  stdin_open: true
  tty: true
 
 
- # optional: default branch (can be overridden per-run)
- environment:
- - BRANCH=${BRANCH:-master}
+ redis:
+ image: redis:7-alpine
+ ports:
+ - "6379:6379"
+
+ celery_worker:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important
 
+ celery_beat:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important

readme.md

@@ -19,4 +19,20 @@ A lightweight, news intelligence service for ingesting free news sources, taggin
 1. **Clone** the repo:
  ```bash
  git clone https://github.com/JustInternetAI/JustInsight.git
- cd justinsight
+ cd justinsight
+
+
+## Setting up Celery Beat:
+- On MacOS
+ - run: docker compose up --build
+ - OR for running in the background: docker compose up -d
+ - use local development section of entrypoint.sh
+ - to shut down celery: docker compose down
+- On the EC2
+ - run: docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build
+ - OR for running in the background: docker compose up -d
+ - use EC2 section of entrypoint.sh
+ - to shut down celery: docker compose down
+
+*When running in background you can use "docker logs <container_name_or_id>" to see the log and "docker ps"
+to find the ID.

requirements.txt

@@ -0,0 +1 @@
+celery[redis]

src/ingest/bbc_rss.py

@@ -1 +1,3 @@
-print("Hello World!")
+
+def testTask():
+ print("Hello World! From the RSS file")

src/justInsight/celery.py

@@ -0,0 +1,18 @@
+# celery.py
+from celery import Celery
+
+app = Celery(
+ "justInsight",
+ broker="redis://redis:6379/0",
+ backend="redis://redis:6379/0",
+ include=["justInsight.tasks"], # adjust this to your tasks file
+)
+
+# Optional beat schedule
+app.conf.beat_schedule = {
+ "sample-task-every-10-seconds": {
+ "task": "justInsight.tasks.sample_task",
+ "schedule": 10.0,
+ "args": (),
+ },
+}

src/justInsight/tasks.py

@@ -0,0 +1,9 @@
+# tasks.py
+from celery import shared_task
+from ingest.bbc_rss import testTask
+
+
+@shared_task
+def sample_task():
+ testTask()
+ return "done!"

.docker/Dockerfile

@@ -6,15 +6,23 @@ RUN apt-get update && \
  apt-get install -y python3 python3-pip git && \
  rm -rf /var/lib/apt/lists/*
 
+# Install requirements
+COPY requirements.txt /workspace/requirements.txt
+RUN pip install --no-cache-dir -r /workspace/requirements.txt
+
 # Set working directory
 WORKDIR /workspace
 
+# Ensure Python can import from this path
+#ENV PYTHONPATH=/workspace/src
+ENV PYTHONPATH=/workspace:/workspace/src
+
 
 # Copy the entrypoint from the .docker folder into the image
 COPY .docker/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
 
-ENTRYPOINT ["/use/local/bin/entrypoint.sh"]
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 CMD ["bash"]
 
 

.docker/entrypoint.sh

@@ -1,32 +1,47 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Default branch (override via env if you want)
-BRANCH="${BRANCH:-master}"
-
-# Tell Git that /workspace is safe to operate on
-git config --global --add safe.directory /workspace
-
-# Bail early if we don’t see a .git folder
-if [ ! -d .git ]; then
- echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
- exit 1
-fi
-
-# Update & switch branches
-echo "⏳ Fetching updates..."
-git fetch origin --prune
-
-echo "⏳ Checking out branch '$BRANCH'..."
-# if branch already exists locally, switch; otherwise track remote
-if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
- git checkout "$BRANCH"
-else
- git checkout --track "origin/$BRANCH"
-fi
-
-echo "⏳ Pulling latest changes..."
-git pull --ff-only origin "$BRANCH"
+#use this code for deployment on EC2
+#######################################################################################
+# # Default branch (override via env if you want)
+# BRANCH="${BRANCH:-master}"
+
+# # Tell Git that /workspace is safe to operate on
+# git config --global --add safe.directory /workspace
+
+# # Bail early if we don’t see a .git folder
+# if [ ! -d .git ]; then
+# echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
+# exit 1
+# fi
+
+# # Update & switch branches
+# echo "⏳ Fetching updates..."
+# git fetch origin --prune
+
+# echo "⏳ Checking out branch '$BRANCH'..."
+# # if branch already exists locally, switch; otherwise track remote
+# if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
+# git checkout "$BRANCH"
+# else
+# git checkout --track "origin/$BRANCH"
+# fi
+
+# echo "⏳ Pulling latest changes..."
+# git pull --ff-only origin "$BRANCH"
+#######################################################################################
+
+
+#use this code for local development
+#######################################################################################
+# Skip Git checkout for local dev — just use the mounted code as-is
+echo "Skipping Git checkout — using mounted code in /workspace"
+
+# Set the Python path and working dir
+#cd /workspace/src
+#export PYTHONPATH=/workspace/src
+#######################################################################################
+
 
 # Hand off to whatever was passed in (e.g. bash)
 exec "${@:-bash}"

.dockerignore

@@ -0,0 +1,4 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*
+.github
+.gitignore

.gitignore

@@ -0,0 +1,2 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*

docker-compose.gpu.yml

@@ -0,0 +1,8 @@
+version: "3.8"
+services:
+ justinsight_service:
+ gpus: all
+ celery_worker:
+ gpus: all
+ celery_beat:
+ gpus: all

docker-compose.yml

@@ -1,25 +1,46 @@
 version: '3.8'
-
+
 services:
  justinsight_service:
  build:
  context: .
  dockerfile: .docker/Dockerfile
- gpus: all
 
  # bind-mount your repo and the shared EBS volume
  volumes:
  - ./:/workspace:cached
  - ./shared:/workspace/shared:cached
 
  # ensure entrypoint.sh runs (clones & checks out the right branch)
- entrypoint: ["/usr/local/bin/entrypoint.sh"]
+#entrypoint: ["/usr/local/bin/entrypoint.sh"]
  command: ["bash"]
  stdin_open: true
  tty: true
 
 
- # optional: default branch (can be overridden per-run)
- environment:
- - BRANCH=${BRANCH:-master}
+ redis:
+ image: redis:7-alpine
+ ports:
+ - "6379:6379"
+
+ celery_worker:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important
 
+ celery_beat:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important

readme.md

@@ -19,4 +19,20 @@ A lightweight, news intelligence service for ingesting free news sources, taggin
 1. **Clone** the repo:
  ```bash
  git clone https://github.com/JustInternetAI/JustInsight.git
- cd justinsight
+ cd justinsight
+
+
+## Setting up Celery Beat:
+- On MacOS
+ - run: docker compose up --build
+ - OR for running in the background: docker compose up -d
+ - use local development section of entrypoint.sh
+ - to shut down celery: docker compose down
+- On the EC2
+ - run: docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build
+ - OR for running in the background: docker compose up -d
+ - use EC2 section of entrypoint.sh
+ - to shut down celery: docker compose down
+
+*When running in background you can use "docker logs <container_name_or_id>" to see the log and "docker ps"
+to find the ID.

requirements.txt

@@ -0,0 +1 @@
+celery[redis]

src/ingest/bbc_rss.py

@@ -1 +1,3 @@
-print("Hello World!")
+
+def testTask():
+ print("Hello World! From the RSS file")

src/justInsight/celery.py

@@ -0,0 +1,18 @@
+# celery.py
+from celery import Celery
+
+app = Celery(
+ "justInsight",
+ broker="redis://redis:6379/0",
+ backend="redis://redis:6379/0",
+ include=["justInsight.tasks"], # adjust this to your tasks file
+)
+
+# Optional beat schedule
+app.conf.beat_schedule = {
+ "sample-task-every-10-seconds": {
+ "task": "justInsight.tasks.sample_task",
+ "schedule": 10.0,
+ "args": (),
+ },
+}

src/justInsight/tasks.py

@@ -0,0 +1,9 @@
+# tasks.py
+from celery import shared_task
+from ingest.bbc_rss import testTask
+
+
+@shared_task
+def sample_task():
+ testTask()
+ return "done!"

.docker/Dockerfile

@@ -6,15 +6,23 @@ RUN apt-get update && \
  apt-get install -y python3 python3-pip git && \
  rm -rf /var/lib/apt/lists/*
 
+# Install requirements
+COPY requirements.txt /workspace/requirements.txt
+RUN pip install --no-cache-dir -r /workspace/requirements.txt
+
 # Set working directory
 WORKDIR /workspace
 
+# Ensure Python can import from this path
+#ENV PYTHONPATH=/workspace/src
+ENV PYTHONPATH=/workspace:/workspace/src
+
 
 # Copy the entrypoint from the .docker folder into the image
 COPY .docker/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
 
-ENTRYPOINT ["/use/local/bin/entrypoint.sh"]
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 CMD ["bash"]
 
 

.docker/entrypoint.sh

@@ -1,32 +1,47 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Default branch (override via env if you want)
-BRANCH="${BRANCH:-master}"
-
-# Tell Git that /workspace is safe to operate on
-git config --global --add safe.directory /workspace
-
-# Bail early if we don’t see a .git folder
-if [ ! -d .git ]; then
- echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
- exit 1
-fi
-
-# Update & switch branches
-echo "⏳ Fetching updates..."
-git fetch origin --prune
-
-echo "⏳ Checking out branch '$BRANCH'..."
-# if branch already exists locally, switch; otherwise track remote
-if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
- git checkout "$BRANCH"
-else
- git checkout --track "origin/$BRANCH"
-fi
-
-echo "⏳ Pulling latest changes..."
-git pull --ff-only origin "$BRANCH"
+#use this code for deployment on EC2
+#######################################################################################
+# # Default branch (override via env if you want)
+# BRANCH="${BRANCH:-master}"
+
+# # Tell Git that /workspace is safe to operate on
+# git config --global --add safe.directory /workspace
+
+# # Bail early if we don’t see a .git folder
+# if [ ! -d .git ]; then
+# echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
+# exit 1
+# fi
+
+# # Update & switch branches
+# echo "⏳ Fetching updates..."
+# git fetch origin --prune
+
+# echo "⏳ Checking out branch '$BRANCH'..."
+# # if branch already exists locally, switch; otherwise track remote
+# if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
+# git checkout "$BRANCH"
+# else
+# git checkout --track "origin/$BRANCH"
+# fi
+
+# echo "⏳ Pulling latest changes..."
+# git pull --ff-only origin "$BRANCH"
+#######################################################################################
+
+
+#use this code for local development
+#######################################################################################
+# Skip Git checkout for local dev — just use the mounted code as-is
+echo "Skipping Git checkout — using mounted code in /workspace"
+
+# Set the Python path and working dir
+#cd /workspace/src
+#export PYTHONPATH=/workspace/src
+#######################################################################################
+
 
 # Hand off to whatever was passed in (e.g. bash)
 exec "${@:-bash}"

.dockerignore

@@ -0,0 +1,4 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*
+.github
+.gitignore

.gitignore

@@ -0,0 +1,2 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*

docker-compose.gpu.yml

@@ -0,0 +1,8 @@
+version: "3.8"
+services:
+ justinsight_service:
+ gpus: all
+ celery_worker:
+ gpus: all
+ celery_beat:
+ gpus: all

docker-compose.yml

@@ -1,25 +1,46 @@
 version: '3.8'
-
+
 services:
  justinsight_service:
  build:
  context: .
  dockerfile: .docker/Dockerfile
- gpus: all
 
  # bind-mount your repo and the shared EBS volume
  volumes:
  - ./:/workspace:cached
  - ./shared:/workspace/shared:cached
 
  # ensure entrypoint.sh runs (clones & checks out the right branch)
- entrypoint: ["/usr/local/bin/entrypoint.sh"]
+#entrypoint: ["/usr/local/bin/entrypoint.sh"]
  command: ["bash"]
  stdin_open: true
  tty: true
 
 
- # optional: default branch (can be overridden per-run)
- environment:
- - BRANCH=${BRANCH:-master}
+ redis:
+ image: redis:7-alpine
+ ports:
+ - "6379:6379"
+
+ celery_worker:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important
 
+ celery_beat:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important

readme.md

@@ -19,4 +19,20 @@ A lightweight, news intelligence service for ingesting free news sources, taggin
 1. **Clone** the repo:
  ```bash
  git clone https://github.com/JustInternetAI/JustInsight.git
- cd justinsight
+ cd justinsight
+
+
+## Setting up Celery Beat:
+- On MacOS
+ - run: docker compose up --build
+ - OR for running in the background: docker compose up -d
+ - use local development section of entrypoint.sh
+ - to shut down celery: docker compose down
+- On the EC2
+ - run: docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build
+ - OR for running in the background: docker compose up -d
+ - use EC2 section of entrypoint.sh
+ - to shut down celery: docker compose down
+
+*When running in background you can use "docker logs <container_name_or_id>" to see the log and "docker ps"
+to find the ID.

requirements.txt

@@ -0,0 +1 @@
+celery[redis]

src/ingest/bbc_rss.py

@@ -1 +1,3 @@
-print("Hello World!")
+
+def testTask():
+ print("Hello World! From the RSS file")

src/justInsight/celery.py

@@ -0,0 +1,18 @@
+# celery.py
+from celery import Celery
+
+app = Celery(
+ "justInsight",
+ broker="redis://redis:6379/0",
+ backend="redis://redis:6379/0",
+ include=["justInsight.tasks"], # adjust this to your tasks file
+)
+
+# Optional beat schedule
+app.conf.beat_schedule = {
+ "sample-task-every-10-seconds": {
+ "task": "justInsight.tasks.sample_task",
+ "schedule": 10.0,
+ "args": (),
+ },
+}

src/justInsight/tasks.py

@@ -0,0 +1,9 @@
+# tasks.py
+from celery import shared_task
+from ingest.bbc_rss import testTask
+
+
+@shared_task
+def sample_task():
+ testTask()
+ return "done!"

.docker/Dockerfile

@@ -6,15 +6,23 @@ RUN apt-get update && \
  apt-get install -y python3 python3-pip git && \
  rm -rf /var/lib/apt/lists/*
 
+# Install requirements
+COPY requirements.txt /workspace/requirements.txt
+RUN pip install --no-cache-dir -r /workspace/requirements.txt
+
 # Set working directory
 WORKDIR /workspace
 
+# Ensure Python can import from this path
+#ENV PYTHONPATH=/workspace/src
+ENV PYTHONPATH=/workspace:/workspace/src
+
 
 # Copy the entrypoint from the .docker folder into the image
 COPY .docker/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
 
-ENTRYPOINT ["/use/local/bin/entrypoint.sh"]
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 CMD ["bash"]
 
 

.docker/entrypoint.sh

@@ -1,32 +1,47 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Default branch (override via env if you want)
-BRANCH="${BRANCH:-master}"
-
-# Tell Git that /workspace is safe to operate on
-git config --global --add safe.directory /workspace
-
-# Bail early if we don’t see a .git folder
-if [ ! -d .git ]; then
- echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
- exit 1
-fi
-
-# Update & switch branches
-echo "⏳ Fetching updates..."
-git fetch origin --prune
-
-echo "⏳ Checking out branch '$BRANCH'..."
-# if branch already exists locally, switch; otherwise track remote
-if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
- git checkout "$BRANCH"
-else
- git checkout --track "origin/$BRANCH"
-fi
-
-echo "⏳ Pulling latest changes..."
-git pull --ff-only origin "$BRANCH"
+#use this code for deployment on EC2
+#######################################################################################
+# # Default branch (override via env if you want)
+# BRANCH="${BRANCH:-master}"
+
+# # Tell Git that /workspace is safe to operate on
+# git config --global --add safe.directory /workspace
+
+# # Bail early if we don’t see a .git folder
+# if [ ! -d .git ]; then
+# echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
+# exit 1
+# fi
+
+# # Update & switch branches
+# echo "⏳ Fetching updates..."
+# git fetch origin --prune
+
+# echo "⏳ Checking out branch '$BRANCH'..."
+# # if branch already exists locally, switch; otherwise track remote
+# if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
+# git checkout "$BRANCH"
+# else
+# git checkout --track "origin/$BRANCH"
+# fi
+
+# echo "⏳ Pulling latest changes..."
+# git pull --ff-only origin "$BRANCH"
+#######################################################################################
+
+
+#use this code for local development
+#######################################################################################
+# Skip Git checkout for local dev — just use the mounted code as-is
+echo "Skipping Git checkout — using mounted code in /workspace"
+
+# Set the Python path and working dir
+#cd /workspace/src
+#export PYTHONPATH=/workspace/src
+#######################################################################################
+
 
 # Hand off to whatever was passed in (e.g. bash)
 exec "${@:-bash}"

.dockerignore

@@ -0,0 +1,4 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*
+.github
+.gitignore

.gitignore

@@ -0,0 +1,2 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*

docker-compose.gpu.yml

@@ -0,0 +1,8 @@
+version: "3.8"
+services:
+ justinsight_service:
+ gpus: all
+ celery_worker:
+ gpus: all
+ celery_beat:
+ gpus: all

docker-compose.yml

@@ -1,25 +1,46 @@
 version: '3.8'
-
+
 services:
  justinsight_service:
  build:
  context: .
  dockerfile: .docker/Dockerfile
- gpus: all
 
  # bind-mount your repo and the shared EBS volume
  volumes:
  - ./:/workspace:cached
  - ./shared:/workspace/shared:cached
 
  # ensure entrypoint.sh runs (clones & checks out the right branch)
- entrypoint: ["/usr/local/bin/entrypoint.sh"]
+#entrypoint: ["/usr/local/bin/entrypoint.sh"]
  command: ["bash"]
  stdin_open: true
  tty: true
 
 
- # optional: default branch (can be overridden per-run)
- environment:
- - BRANCH=${BRANCH:-master}
+ redis:
+ image: redis:7-alpine
+ ports:
+ - "6379:6379"
+
+ celery_worker:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important
 
+ celery_beat:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important

readme.md

@@ -19,4 +19,20 @@ A lightweight, news intelligence service for ingesting free news sources, taggin
 1. **Clone** the repo:
  ```bash
  git clone https://github.com/JustInternetAI/JustInsight.git
- cd justinsight
+ cd justinsight
+
+
+## Setting up Celery Beat:
+- On MacOS
+ - run: docker compose up --build
+ - OR for running in the background: docker compose up -d
+ - use local development section of entrypoint.sh
+ - to shut down celery: docker compose down
+- On the EC2
+ - run: docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build
+ - OR for running in the background: docker compose up -d
+ - use EC2 section of entrypoint.sh
+ - to shut down celery: docker compose down
+
+*When running in background you can use "docker logs <container_name_or_id>" to see the log and "docker ps"
+to find the ID.

requirements.txt

@@ -0,0 +1 @@
+celery[redis]

src/ingest/bbc_rss.py

@@ -1 +1,3 @@
-print("Hello World!")
+
+def testTask():
+ print("Hello World! From the RSS file")

src/justInsight/celery.py

@@ -0,0 +1,18 @@
+# celery.py
+from celery import Celery
+
+app = Celery(
+ "justInsight",
+ broker="redis://redis:6379/0",
+ backend="redis://redis:6379/0",
+ include=["justInsight.tasks"], # adjust this to your tasks file
+)
+
+# Optional beat schedule
+app.conf.beat_schedule = {
+ "sample-task-every-10-seconds": {
+ "task": "justInsight.tasks.sample_task",
+ "schedule": 10.0,
+ "args": (),
+ },
+}

src/justInsight/tasks.py

@@ -0,0 +1,9 @@
+# tasks.py
+from celery import shared_task
+from ingest.bbc_rss import testTask
+
+
+@shared_task
+def sample_task():
+ testTask()
+ return "done!"

.docker/Dockerfile

@@ -6,15 +6,23 @@ RUN apt-get update && \
  apt-get install -y python3 python3-pip git && \
  rm -rf /var/lib/apt/lists/*
 
+# Install requirements
+COPY requirements.txt /workspace/requirements.txt
+RUN pip install --no-cache-dir -r /workspace/requirements.txt
+
 # Set working directory
 WORKDIR /workspace
 
+# Ensure Python can import from this path
+#ENV PYTHONPATH=/workspace/src
+ENV PYTHONPATH=/workspace:/workspace/src
+
 
 # Copy the entrypoint from the .docker folder into the image
 COPY .docker/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
 
-ENTRYPOINT ["/use/local/bin/entrypoint.sh"]
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 CMD ["bash"]
 
 

.docker/entrypoint.sh

@@ -1,32 +1,47 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Default branch (override via env if you want)
-BRANCH="${BRANCH:-master}"
-
-# Tell Git that /workspace is safe to operate on
-git config --global --add safe.directory /workspace
-
-# Bail early if we don’t see a .git folder
-if [ ! -d .git ]; then
- echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
- exit 1
-fi
-
-# Update & switch branches
-echo "⏳ Fetching updates..."
-git fetch origin --prune
-
-echo "⏳ Checking out branch '$BRANCH'..."
-# if branch already exists locally, switch; otherwise track remote
-if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
- git checkout "$BRANCH"
-else
- git checkout --track "origin/$BRANCH"
-fi
-
-echo "⏳ Pulling latest changes..."
-git pull --ff-only origin "$BRANCH"
+#use this code for deployment on EC2
+#######################################################################################
+# # Default branch (override via env if you want)
+# BRANCH="${BRANCH:-master}"
+
+# # Tell Git that /workspace is safe to operate on
+# git config --global --add safe.directory /workspace
+
+# # Bail early if we don’t see a .git folder
+# if [ ! -d .git ]; then
+# echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
+# exit 1
+# fi
+
+# # Update & switch branches
+# echo "⏳ Fetching updates..."
+# git fetch origin --prune
+
+# echo "⏳ Checking out branch '$BRANCH'..."
+# # if branch already exists locally, switch; otherwise track remote
+# if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
+# git checkout "$BRANCH"
+# else
+# git checkout --track "origin/$BRANCH"
+# fi
+
+# echo "⏳ Pulling latest changes..."
+# git pull --ff-only origin "$BRANCH"
+#######################################################################################
+
+
+#use this code for local development
+#######################################################################################
+# Skip Git checkout for local dev — just use the mounted code as-is
+echo "Skipping Git checkout — using mounted code in /workspace"
+
+# Set the Python path and working dir
+#cd /workspace/src
+#export PYTHONPATH=/workspace/src
+#######################################################################################
+
 
 # Hand off to whatever was passed in (e.g. bash)
 exec "${@:-bash}"

.dockerignore

@@ -0,0 +1,4 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*
+.github
+.gitignore

.gitignore

@@ -0,0 +1,2 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*

docker-compose.gpu.yml

@@ -0,0 +1,8 @@
+version: "3.8"
+services:
+ justinsight_service:
+ gpus: all
+ celery_worker:
+ gpus: all
+ celery_beat:
+ gpus: all

docker-compose.yml

@@ -1,25 +1,46 @@
 version: '3.8'
-
+
 services:
  justinsight_service:
  build:
  context: .
  dockerfile: .docker/Dockerfile
- gpus: all
 
  # bind-mount your repo and the shared EBS volume
  volumes:
  - ./:/workspace:cached
  - ./shared:/workspace/shared:cached
 
  # ensure entrypoint.sh runs (clones & checks out the right branch)
- entrypoint: ["/usr/local/bin/entrypoint.sh"]
+#entrypoint: ["/usr/local/bin/entrypoint.sh"]
  command: ["bash"]
  stdin_open: true
  tty: true
 
 
- # optional: default branch (can be overridden per-run)
- environment:
- - BRANCH=${BRANCH:-master}
+ redis:
+ image: redis:7-alpine
+ ports:
+ - "6379:6379"
+
+ celery_worker:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important
 
+ celery_beat:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important

readme.md

@@ -19,4 +19,20 @@ A lightweight, news intelligence service for ingesting free news sources, taggin
 1. **Clone** the repo:
  ```bash
  git clone https://github.com/JustInternetAI/JustInsight.git
- cd justinsight
+ cd justinsight
+
+
+## Setting up Celery Beat:
+- On MacOS
+ - run: docker compose up --build
+ - OR for running in the background: docker compose up -d
+ - use local development section of entrypoint.sh
+ - to shut down celery: docker compose down
+- On the EC2
+ - run: docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build
+ - OR for running in the background: docker compose up -d
+ - use EC2 section of entrypoint.sh
+ - to shut down celery: docker compose down
+
+*When running in background you can use "docker logs <container_name_or_id>" to see the log and "docker ps"
+to find the ID.

requirements.txt

@@ -0,0 +1 @@
+celery[redis]

src/ingest/bbc_rss.py

@@ -1 +1,3 @@
-print("Hello World!")
+
+def testTask():
+ print("Hello World! From the RSS file")

src/justInsight/celery.py

@@ -0,0 +1,18 @@
+# celery.py
+from celery import Celery
+
+app = Celery(
+ "justInsight",
+ broker="redis://redis:6379/0",
+ backend="redis://redis:6379/0",
+ include=["justInsight.tasks"], # adjust this to your tasks file
+)
+
+# Optional beat schedule
+app.conf.beat_schedule = {
+ "sample-task-every-10-seconds": {
+ "task": "justInsight.tasks.sample_task",
+ "schedule": 10.0,
+ "args": (),
+ },
+}

src/justInsight/tasks.py

@@ -0,0 +1,9 @@
+# tasks.py
+from celery import shared_task
+from ingest.bbc_rss import testTask
+
+
+@shared_task
+def sample_task():
+ testTask()
+ return "done!"

.docker/Dockerfile

@@ -6,15 +6,23 @@ RUN apt-get update && \
  apt-get install -y python3 python3-pip git && \
  rm -rf /var/lib/apt/lists/*
 
+# Install requirements
+COPY requirements.txt /workspace/requirements.txt
+RUN pip install --no-cache-dir -r /workspace/requirements.txt
+
 # Set working directory
 WORKDIR /workspace
 
+# Ensure Python can import from this path
+#ENV PYTHONPATH=/workspace/src
+ENV PYTHONPATH=/workspace:/workspace/src
+
 
 # Copy the entrypoint from the .docker folder into the image
 COPY .docker/entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
 
-ENTRYPOINT ["/use/local/bin/entrypoint.sh"]
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 CMD ["bash"]
 
 

.docker/entrypoint.sh

@@ -1,32 +1,47 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Default branch (override via env if you want)
-BRANCH="${BRANCH:-master}"
-
-# Tell Git that /workspace is safe to operate on
-git config --global --add safe.directory /workspace
-
-# Bail early if we don’t see a .git folder
-if [ ! -d .git ]; then
- echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
- exit 1
-fi
-
-# Update & switch branches
-echo "⏳ Fetching updates..."
-git fetch origin --prune
-
-echo "⏳ Checking out branch '$BRANCH'..."
-# if branch already exists locally, switch; otherwise track remote
-if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
- git checkout "$BRANCH"
-else
- git checkout --track "origin/$BRANCH"
-fi
-
-echo "⏳ Pulling latest changes..."
-git pull --ff-only origin "$BRANCH"
+#use this code for deployment on EC2
+#######################################################################################
+# # Default branch (override via env if you want)
+# BRANCH="${BRANCH:-master}"
+
+# # Tell Git that /workspace is safe to operate on
+# git config --global --add safe.directory /workspace
+
+# # Bail early if we don’t see a .git folder
+# if [ ! -d .git ]; then
+# echo "❌ Error: /workspace is not a git repo. Please 'git clone' on the host first."
+# exit 1
+# fi
+
+# # Update & switch branches
+# echo "⏳ Fetching updates..."
+# git fetch origin --prune
+
+# echo "⏳ Checking out branch '$BRANCH'..."
+# # if branch already exists locally, switch; otherwise track remote
+# if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then
+# git checkout "$BRANCH"
+# else
+# git checkout --track "origin/$BRANCH"
+# fi
+
+# echo "⏳ Pulling latest changes..."
+# git pull --ff-only origin "$BRANCH"
+#######################################################################################
+
+
+#use this code for local development
+#######################################################################################
+# Skip Git checkout for local dev — just use the mounted code as-is
+echo "Skipping Git checkout — using mounted code in /workspace"
+
+# Set the Python path and working dir
+#cd /workspace/src
+#export PYTHONPATH=/workspace/src
+#######################################################################################
+
 
 # Hand off to whatever was passed in (e.g. bash)
 exec "${@:-bash}"

.dockerignore

@@ -0,0 +1,4 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*
+.github
+.gitignore

.gitignore

@@ -0,0 +1,2 @@
+celerybeat-schedule.*
+src/celerybeat-schedule.*

docker-compose.gpu.yml

@@ -0,0 +1,8 @@
+version: "3.8"
+services:
+ justinsight_service:
+ gpus: all
+ celery_worker:
+ gpus: all
+ celery_beat:
+ gpus: all

docker-compose.yml

@@ -1,25 +1,46 @@
 version: '3.8'
-
+
 services:
  justinsight_service:
  build:
  context: .
  dockerfile: .docker/Dockerfile
- gpus: all
 
  # bind-mount your repo and the shared EBS volume
  volumes:
  - ./:/workspace:cached
  - ./shared:/workspace/shared:cached
 
  # ensure entrypoint.sh runs (clones & checks out the right branch)
- entrypoint: ["/usr/local/bin/entrypoint.sh"]
+#entrypoint: ["/usr/local/bin/entrypoint.sh"]
  command: ["bash"]
  stdin_open: true
  tty: true
 
 
- # optional: default branch (can be overridden per-run)
- environment:
- - BRANCH=${BRANCH:-master}
+ redis:
+ image: redis:7-alpine
+ ports:
+ - "6379:6379"
+
+ celery_worker:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important
 
+ celery_beat:
+ build:
+ context: .
+ dockerfile: .docker/Dockerfile
+ volumes:
+ - .:/workspace:cached
+ depends_on:
+ - redis
+ #entrypoint: ["/usr/local/bin/entrypoint.sh"]
+ command: ["celery", "-A", justinsight.celery, "beat", "--loglevel=info"] #the lowercase j is actually so important

readme.md

@@ -19,4 +19,20 @@ A lightweight, news intelligence service for ingesting free news sources, taggin
 1. **Clone** the repo:
  ```bash
  git clone https://github.com/JustInternetAI/JustInsight.git
- cd justinsight
+ cd justinsight
+
+
+## Setting up Celery Beat:
+- On MacOS
+ - run: docker compose up --build
+ - OR for running in the background: docker compose up -d
+ - use local development section of entrypoint.sh
+ - to shut down celery: docker compose down
+- On the EC2
+ - run: docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build
+ - OR for running in the background: docker compose up -d
+ - use EC2 section of entrypoint.sh
+ - to shut down celery: docker compose down
+
+*When running in background you can use "docker logs <container_name_or_id>" to see the log and "docker ps"
+to find the ID.

requirements.txt

@@ -0,0 +1 @@
+celery[redis]

src/ingest/bbc_rss.py

@@ -1 +1,3 @@
-print("Hello World!")
+
+def testTask():
+ print("Hello World! From the RSS file")

src/justInsight/celery.py

@@ -0,0 +1,18 @@
+# celery.py
+from celery import Celery
+
+app = Celery(
+ "justInsight",
+ broker="redis://redis:6379/0",
+ backend="redis://redis:6379/0",
+ include=["justInsight.tasks"], # adjust this to your tasks file
+)
+
+# Optional beat schedule
+app.conf.beat_schedule = {
+ "sample-task-every-10-seconds": {
+ "task": "justInsight.tasks.sample_task",
+ "schedule": 10.0,
+ "args": (),
+ },
+}

src/justInsight/tasks.py

@@ -0,0 +1,9 @@
+# tasks.py
+from celery import shared_task
+from ingest.bbc_rss import testTask
+
+
+@shared_task
+def sample_task():
+ testTask()
+ return "done!"