diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 9ecc858..d00a424 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -6,15 +6,23 @@ RUN apt-get update && \ apt-get install -y python3 python3-pip git && \ rm -rf /var/lib/apt/lists/* +# Install requirements +COPY requirements.txt /workspace/requirements.txt +RUN pip install --no-cache-dir -r /workspace/requirements.txt + # Set working directory WORKDIR /workspace +# Ensure Python can import from this path +#ENV PYTHONPATH=/workspace/src +ENV PYTHONPATH=/workspace:/workspace/src + # Copy the entrypoint from the .docker folder into the image COPY .docker/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh -ENTRYPOINT ["/use/local/bin/entrypoint.sh"] +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] CMD ["bash"] diff --git a/.docker/entrypoint.sh b/.docker/entrypoint.sh index 51b26ba..2d62628 100644 --- a/.docker/entrypoint.sh +++ b/.docker/entrypoint.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash set -euo pipefail -# Hand off to whatever was passed in (e.g. bash) exec "${@:-bash}" diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..477abcd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +celerybeat-schedule.* +src/celerybeat-schedule.* +.github +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..96dd22a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +celerybeat-schedule.* +src/celerybeat-schedule.* \ No newline at end of file diff --git a/docker-compose.gpu.yml b/docker-compose.gpu.yml new file mode 100644 index 0000000..bda9d46 --- /dev/null +++ b/docker-compose.gpu.yml @@ -0,0 +1,8 @@ +version: "3.8" +services: + justinsight_service: + gpus: all + celery_worker: + gpus: all + celery_beat: + gpus: all diff --git a/docker-compose.yml b/docker-compose.yml index 7d18737..3e59160 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,10 @@ version: '3.8' - + services: justinsight_service: build: context: . dockerfile: .docker/Dockerfile - gpus: all # bind-mount your repo and the shared EBS volume volumes: @@ -13,10 +12,25 @@ services: - ./shared:/workspace/shared:cached # ensure entrypoint.sh runs (clones & checks out the right branch) - entrypoint: ["/usr/local/bin/entrypoint.sh"] + #entrypoint: ["/usr/local/bin/entrypoint.sh"] command: ["bash"] stdin_open: true tty: true + + redis: + image: redis:7-alpine + ports: + - "6379:6379" - + celery_worker: + build: + context: . + dockerfile: .docker/Dockerfile + volumes: + - .:/workspace:cached + depends_on: + - redis + #entrypoint: ["/usr/local/bin/entrypoint.sh"] + command: ["celery", "-A", justinsight.celery, "worker", "--loglevel=info"] #the lowercase j is actually so important +======= diff --git a/readme.md b/readme.md index 455c2a5..c52820d 100644 --- a/readme.md +++ b/readme.md @@ -19,4 +19,20 @@ A lightweight, news intelligence service for ingesting free news sources, taggin 1. **Clone** the repo: ```bash git clone https://github.com/JustInternetAI/JustInsight.git - cd justinsight \ No newline at end of file + cd justinsight + + +## Setting up Celery Beat: +- On MacOS + - run: docker compose up --build + - OR for running in the background: docker compose up -d + - use local development section of entrypoint.sh + - to shut down celery: docker compose down +- On the EC2 + - run: docker compose -f docker-compose.yml -f docker-compose.gpu.yml up --build + - OR for running in the background: docker compose up -d + - use EC2 section of entrypoint.sh + - to shut down celery: docker compose down + +*When running in background you can use "docker logs " to see the log and "docker ps" +to find the ID. \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e69de29..1ce978e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1 @@ +celery[redis] \ No newline at end of file diff --git a/src/__init__.py b/src/ingest/__init__.py similarity index 100% rename from src/__init__.py rename to src/ingest/__init__.py diff --git a/src/ingest/__pycache__/__init__.cpython-310.pyc b/src/ingest/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..5817e24 Binary files /dev/null and b/src/ingest/__pycache__/__init__.cpython-310.pyc differ diff --git a/src/ingest/__pycache__/bbc_rss.cpython-310.pyc b/src/ingest/__pycache__/bbc_rss.cpython-310.pyc new file mode 100644 index 0000000..acc5de4 Binary files /dev/null and b/src/ingest/__pycache__/bbc_rss.cpython-310.pyc differ diff --git a/src/ingest/bbc_rss.py b/src/ingest/bbc_rss.py index 1dc45ac..ff7e4a7 100644 --- a/src/ingest/bbc_rss.py +++ b/src/ingest/bbc_rss.py @@ -1 +1,3 @@ -print("Hello World!") \ No newline at end of file + +def testTask(): + print("Hello World! From the RSS file") \ No newline at end of file diff --git a/src/justInsight/__init__.py b/src/justInsight/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/justInsight/__pycache__/__init__.cpython-310.pyc b/src/justInsight/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..3dee5e1 Binary files /dev/null and b/src/justInsight/__pycache__/__init__.cpython-310.pyc differ diff --git a/src/justInsight/__pycache__/celery.cpython-310.pyc b/src/justInsight/__pycache__/celery.cpython-310.pyc new file mode 100644 index 0000000..7fa8fd6 Binary files /dev/null and b/src/justInsight/__pycache__/celery.cpython-310.pyc differ diff --git a/src/justInsight/__pycache__/tasks.cpython-310.pyc b/src/justInsight/__pycache__/tasks.cpython-310.pyc new file mode 100644 index 0000000..81b5e86 Binary files /dev/null and b/src/justInsight/__pycache__/tasks.cpython-310.pyc differ diff --git a/src/justInsight/celery.py b/src/justInsight/celery.py new file mode 100644 index 0000000..5a92454 --- /dev/null +++ b/src/justInsight/celery.py @@ -0,0 +1,18 @@ +# celery.py +from celery import Celery + +app = Celery( + "justInsight", + broker="redis://redis:6379/0", + backend="redis://redis:6379/0", + include=["justInsight.tasks"], # adjust this to your tasks file +) + +# Optional beat schedule +app.conf.beat_schedule = { + "sample-task-every-10-seconds": { + "task": "justInsight.tasks.sample_task", + "schedule": 10.0, + "args": (), + }, +} diff --git a/src/justInsight/tasks.py b/src/justInsight/tasks.py new file mode 100644 index 0000000..9378f65 --- /dev/null +++ b/src/justInsight/tasks.py @@ -0,0 +1,9 @@ +# tasks.py +from celery import shared_task +from ingest.bbc_rss import testTask + + +@shared_task +def sample_task(): + testTask() + return "done!" \ No newline at end of file