Integrate Celery task queue for background task processing (#129)
This commit is contained in:
committed by
GitHub
parent
b8ff534ad8
commit
f2dacf4ee0
@@ -45,6 +45,8 @@ services:
|
||||
GROK_ENABLED: "${GROK_ENABLED:-false}"
|
||||
XAI_API_KEY: "${XAI_API_KEY:-}"
|
||||
GROK_DEFAULT_MODEL: "${GROK_DEFAULT_MODEL:-grok-3-fast}"
|
||||
# Celery/Redis — background task queue
|
||||
REDIS_URL: "redis://redis:6379/0"
|
||||
# Taskosaur API — dashboard can reach it on the internal network
|
||||
TASKOSAUR_API_URL: "http://taskosaur:3000/api"
|
||||
extra_hosts:
|
||||
@@ -131,6 +133,30 @@ services:
|
||||
retries: 5
|
||||
start_period: 5s
|
||||
|
||||
# ── Celery Worker — background task processing ──────────────────────────
|
||||
celery-worker:
|
||||
build: .
|
||||
image: timmy-time:latest
|
||||
container_name: timmy-celery-worker
|
||||
user: "0:0"
|
||||
command: ["celery", "-A", "infrastructure.celery.app", "worker", "--loglevel=info", "--concurrency=2"]
|
||||
volumes:
|
||||
- timmy-data:/app/data
|
||||
- ./src:/app/src
|
||||
environment:
|
||||
REDIS_URL: "redis://redis:6379/0"
|
||||
OLLAMA_URL: "${OLLAMA_URL:-http://host.docker.internal:11434}"
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- timmy-net
|
||||
restart: unless-stopped
|
||||
profiles:
|
||||
- celery
|
||||
|
||||
# ── OpenFang — vendored agent runtime sidecar ────────────────────────────
|
||||
openfang:
|
||||
build:
|
||||
|
||||
266
poetry.lock
generated
266
poetry.lock
generated
@@ -391,6 +391,22 @@ torch = "*"
|
||||
tqdm = "*"
|
||||
transformers = "*"
|
||||
|
||||
[[package]]
|
||||
name = "amqp"
|
||||
version = "5.3.1"
|
||||
description = "Low-level AMQP client for Python (fork of amqplib)."
|
||||
optional = true
|
||||
python-versions = ">=3.6"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2"},
|
||||
{file = "amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
vine = ">=5.0.0,<6.0.0"
|
||||
|
||||
[[package]]
|
||||
name = "annotated-doc"
|
||||
version = "0.0.4"
|
||||
@@ -520,6 +536,77 @@ files = [
|
||||
{file = "audioop_lts-0.2.2.tar.gz", hash = "sha256:64d0c62d88e67b98a1a5e71987b7aa7b5bcffc7dcee65b635823dbdd0a8dbbd0"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "billiard"
|
||||
version = "4.2.4"
|
||||
description = "Python multiprocessing fork with improvements and bugfixes"
|
||||
optional = true
|
||||
python-versions = ">=3.7"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5"},
|
||||
{file = "billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "celery"
|
||||
version = "5.3.1"
|
||||
description = "Distributed Task Queue."
|
||||
optional = true
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "celery-5.3.1-py3-none-any.whl", hash = "sha256:27f8f3f3b58de6e0ab4f174791383bbd7445aff0471a43e99cfd77727940753f"},
|
||||
{file = "celery-5.3.1.tar.gz", hash = "sha256:f84d1c21a1520c116c2b7d26593926581191435a03aa74b77c941b93ca1c6210"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
billiard = ">=4.1.0,<5.0"
|
||||
click = ">=8.1.2,<9.0"
|
||||
click-didyoumean = ">=0.3.0"
|
||||
click-plugins = ">=1.1.1"
|
||||
click-repl = ">=0.2.0"
|
||||
kombu = ">=5.3.1,<6.0"
|
||||
python-dateutil = ">=2.8.2"
|
||||
redis = {version = ">=4.5.2,<4.5.5 || >4.5.5", optional = true, markers = "extra == \"redis\""}
|
||||
tzdata = ">=2022.7"
|
||||
vine = ">=5.0.0,<6.0"
|
||||
|
||||
[package.extras]
|
||||
arangodb = ["pyArango (>=2.0.1)"]
|
||||
auth = ["cryptography (==41.0.1)"]
|
||||
azureblockblob = ["azure-storage-blob (>=12.15.0)"]
|
||||
brotli = ["brotli (>=1.0.0) ; platform_python_implementation == \"CPython\"", "brotlipy (>=0.7.0) ; platform_python_implementation == \"PyPy\""]
|
||||
cassandra = ["cassandra-driver (>=3.25.0,<4)"]
|
||||
consul = ["python-consul2 (==0.1.5)"]
|
||||
cosmosdbsql = ["pydocumentdb (==2.3.5)"]
|
||||
couchbase = ["couchbase (>=3.0.0) ; platform_python_implementation != \"PyPy\" and (platform_system != \"Windows\" or python_version < \"3.10\")"]
|
||||
couchdb = ["pycouchdb (==1.14.2)"]
|
||||
django = ["Django (>=2.2.28)"]
|
||||
dynamodb = ["boto3 (>=1.26.143)"]
|
||||
elasticsearch = ["elasticsearch (<8.0)"]
|
||||
eventlet = ["eventlet (>=0.32.0) ; python_version < \"3.10\""]
|
||||
gevent = ["gevent (>=1.5.0)"]
|
||||
librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""]
|
||||
memcache = ["pylibmc (==1.6.3) ; platform_system != \"Windows\""]
|
||||
mongodb = ["pymongo[srv] (>=4.0.2)"]
|
||||
msgpack = ["msgpack (==1.0.5)"]
|
||||
pymemcache = ["python-memcached (==1.59)"]
|
||||
pyro = ["pyro4 (==4.82) ; python_version < \"3.11\""]
|
||||
pytest = ["pytest-celery (==0.0.0)"]
|
||||
redis = ["redis (>=4.5.2,!=4.5.5)"]
|
||||
s3 = ["boto3 (>=1.26.143)"]
|
||||
slmq = ["softlayer-messaging (>=1.0.3)"]
|
||||
solar = ["ephem (==4.1.4) ; platform_python_implementation != \"PyPy\""]
|
||||
sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"]
|
||||
sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.3.0)", "pycurl (>=7.43.0.5) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\"", "urllib3 (>=1.26.16)"]
|
||||
tblib = ["tblib (>=1.3.0) ; python_version < \"3.8.0\"", "tblib (>=1.5.0) ; python_version >= \"3.8.0\""]
|
||||
yaml = ["PyYAML (>=3.10)"]
|
||||
zookeeper = ["kazoo (>=1.3.1)"]
|
||||
zstd = ["zstandard (==0.21.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.2.25"
|
||||
@@ -768,6 +855,61 @@ files = [
|
||||
[package.dependencies]
|
||||
colorama = {version = "*", markers = "platform_system == \"Windows\""}
|
||||
|
||||
[[package]]
|
||||
name = "click-didyoumean"
|
||||
version = "0.3.1"
|
||||
description = "Enables git-like *did-you-mean* feature in click"
|
||||
optional = true
|
||||
python-versions = ">=3.6.2"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c"},
|
||||
{file = "click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
click = ">=7"
|
||||
|
||||
[[package]]
|
||||
name = "click-plugins"
|
||||
version = "1.1.1.2"
|
||||
description = "An extension module for click to enable registering CLI commands via setuptools entry-points."
|
||||
optional = true
|
||||
python-versions = "*"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6"},
|
||||
{file = "click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
click = ">=4.0"
|
||||
|
||||
[package.extras]
|
||||
dev = ["coveralls", "pytest (>=3.6)", "pytest-cov", "wheel"]
|
||||
|
||||
[[package]]
|
||||
name = "click-repl"
|
||||
version = "0.3.0"
|
||||
description = "REPL plugin for Click"
|
||||
optional = true
|
||||
python-versions = ">=3.6"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9"},
|
||||
{file = "click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
click = ">=7.0"
|
||||
prompt-toolkit = ">=3.0.36"
|
||||
|
||||
[package.extras]
|
||||
testing = ["pytest (>=7.2.1)", "pytest-cov (>=4.0.0)", "tox (>=4.4.3)"]
|
||||
|
||||
[[package]]
|
||||
name = "colorama"
|
||||
version = "0.4.6"
|
||||
@@ -1762,6 +1904,43 @@ files = [
|
||||
{file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kombu"
|
||||
version = "5.6.2"
|
||||
description = "Messaging library for Python."
|
||||
optional = true
|
||||
python-versions = ">=3.9"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93"},
|
||||
{file = "kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
amqp = ">=5.1.1,<6.0.0"
|
||||
packaging = "*"
|
||||
tzdata = ">=2025.2"
|
||||
vine = "5.1.0"
|
||||
|
||||
[package.extras]
|
||||
azureservicebus = ["azure-servicebus (>=7.10.0)"]
|
||||
azurestoragequeues = ["azure-identity (>=1.12.0)", "azure-storage-queue (>=12.6.0)"]
|
||||
confluentkafka = ["confluent-kafka (>=2.2.0)"]
|
||||
consul = ["python-consul2 (==0.1.5)"]
|
||||
gcpubsub = ["google-cloud-monitoring (>=2.16.0)", "google-cloud-pubsub (>=2.18.4)", "grpcio (==1.75.1)", "protobuf (==6.32.1)"]
|
||||
librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""]
|
||||
mongodb = ["pymongo (==4.15.3)"]
|
||||
msgpack = ["msgpack (==1.1.2)"]
|
||||
pyro = ["pyro4 (==4.82)"]
|
||||
qpid = ["qpid-python (==1.36.0.post1)", "qpid-tools (==1.36.0.post1)"]
|
||||
redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2,<6.5)"]
|
||||
slmq = ["softlayer_messaging (>=1.0.3)"]
|
||||
sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"]
|
||||
sqs = ["boto3 (>=1.26.143)", "pycurl (>=7.43.0.5) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\"", "urllib3 (>=1.26.16)"]
|
||||
yaml = ["PyYAML (>=3.10)"]
|
||||
zookeeper = ["kazoo (>=2.8.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "markdown-it-py"
|
||||
version = "4.0.0"
|
||||
@@ -2694,6 +2873,22 @@ files = [
|
||||
[package.dependencies]
|
||||
tqdm = "*"
|
||||
|
||||
[[package]]
|
||||
name = "prompt-toolkit"
|
||||
version = "3.0.52"
|
||||
description = "Library for building powerful interactive command lines in Python"
|
||||
optional = true
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955"},
|
||||
{file = "prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
wcwidth = "*"
|
||||
|
||||
[[package]]
|
||||
name = "propcache"
|
||||
version = "0.4.1"
|
||||
@@ -6771,6 +6966,22 @@ psutil = ["psutil (>=3.0)"]
|
||||
setproctitle = ["setproctitle"]
|
||||
testing = ["filelock"]
|
||||
|
||||
[[package]]
|
||||
name = "python-dateutil"
|
||||
version = "2.9.0.post0"
|
||||
description = "Extensions to the standard Python datetime module"
|
||||
optional = true
|
||||
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"},
|
||||
{file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
six = ">=1.5"
|
||||
|
||||
[[package]]
|
||||
name = "python-dotenv"
|
||||
version = "1.2.1"
|
||||
@@ -7420,6 +7631,19 @@ files = [
|
||||
{file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "six"
|
||||
version = "1.17.0"
|
||||
description = "Python 2 and 3 compatibility utilities"
|
||||
optional = true
|
||||
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"},
|
||||
{file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smmap"
|
||||
version = "5.0.2"
|
||||
@@ -7912,6 +8136,19 @@ files = [
|
||||
[package.dependencies]
|
||||
typing-extensions = ">=4.12.0"
|
||||
|
||||
[[package]]
|
||||
name = "tzdata"
|
||||
version = "2025.3"
|
||||
description = "Provider of IANA time zone data"
|
||||
optional = true
|
||||
python-versions = ">=2"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1"},
|
||||
{file = "tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "urllib3"
|
||||
version = "2.6.3"
|
||||
@@ -8024,6 +8261,19 @@ dev = ["Cython (>=3.0,<4.0)", "setuptools (>=60)"]
|
||||
docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx_rtd_theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
|
||||
test = ["aiohttp (>=3.10.5)", "flake8 (>=6.1,<7.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=25.3.0,<25.4.0)", "pycodestyle (>=2.11.0,<2.12.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "vine"
|
||||
version = "5.1.0"
|
||||
description = "Python promises."
|
||||
optional = true
|
||||
python-versions = ">=3.6"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc"},
|
||||
{file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "watchfiles"
|
||||
version = "1.1.1"
|
||||
@@ -8146,6 +8396,19 @@ files = [
|
||||
[package.dependencies]
|
||||
anyio = ">=3.0.0"
|
||||
|
||||
[[package]]
|
||||
name = "wcwidth"
|
||||
version = "0.6.0"
|
||||
description = "Measures the displayed width of unicode strings in a terminal"
|
||||
optional = true
|
||||
python-versions = ">=3.8"
|
||||
groups = ["main"]
|
||||
markers = "extra == \"celery\""
|
||||
files = [
|
||||
{file = "wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad"},
|
||||
{file = "wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "websocket-client"
|
||||
version = "1.9.0"
|
||||
@@ -8399,6 +8662,7 @@ propcache = ">=0.2.1"
|
||||
|
||||
[extras]
|
||||
bigbrain = ["airllm"]
|
||||
celery = ["celery"]
|
||||
dev = ["pytest", "pytest-asyncio", "pytest-cov", "pytest-randomly", "pytest-timeout", "pytest-xdist", "selenium"]
|
||||
discord = ["discord.py"]
|
||||
telegram = ["python-telegram-bot"]
|
||||
@@ -8407,4 +8671,4 @@ voice = ["pyttsx3"]
|
||||
[metadata]
|
||||
lock-version = "2.1"
|
||||
python-versions = ">=3.11,<4"
|
||||
content-hash = "c4a7adbe5b16d5ea5b0d8425ca9373dfa8b20f0bc1b3a9ad90581e0a005e7acd"
|
||||
content-hash = "337367c3d31512dfd2600ed1994b4c42a8c961d4eea2ced02a5492dbddd70faf"
|
||||
|
||||
@@ -41,6 +41,7 @@ sentence-transformers = ">=2.0.0" # Local embeddings for brain
|
||||
numpy = ">=1.24.0"
|
||||
# Optional extras
|
||||
redis = { version = ">=5.0.0", optional = true }
|
||||
celery = { version = ">=5.3.0", extras = ["redis"], optional = true }
|
||||
python-telegram-bot = { version = ">=21.0", optional = true }
|
||||
"discord.py" = { version = ">=2.3.0", optional = true }
|
||||
airllm = { version = ">=2.9.0", optional = true }
|
||||
@@ -58,6 +59,7 @@ telegram = ["python-telegram-bot"]
|
||||
discord = ["discord.py"]
|
||||
bigbrain = ["airllm"]
|
||||
voice = ["pyttsx3"]
|
||||
celery = ["celery"]
|
||||
dev = ["pytest", "pytest-asyncio", "pytest-cov", "pytest-timeout", "pytest-randomly", "pytest-xdist", "selenium"]
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
|
||||
@@ -23,6 +23,10 @@ class Settings(BaseSettings):
|
||||
# Discord bot token — set via DISCORD_TOKEN env var or the /discord/setup endpoint
|
||||
discord_token: str = ""
|
||||
|
||||
# ── Celery / Redis ──────────────────────────────────────────────────────
|
||||
redis_url: str = "redis://localhost:6379/0"
|
||||
celery_enabled: bool = True
|
||||
|
||||
# ── AirLLM / backend selection ───────────────────────────────────────────
|
||||
# "ollama" — always use Ollama (default, safe everywhere)
|
||||
# "airllm" — always use AirLLM (requires pip install ".[bigbrain]")
|
||||
|
||||
@@ -41,6 +41,7 @@ from dashboard.routes.thinking import router as thinking_router
|
||||
from dashboard.routes.calm import router as calm_router
|
||||
from dashboard.routes.swarm import router as swarm_router
|
||||
from dashboard.routes.system import router as system_router
|
||||
from dashboard.routes.tasks_celery import router as celery_router
|
||||
from infrastructure.router.api import router as cascade_router
|
||||
|
||||
# Import dedicated middleware
|
||||
@@ -306,6 +307,7 @@ app.include_router(thinking_router)
|
||||
app.include_router(calm_router)
|
||||
app.include_router(swarm_router)
|
||||
app.include_router(system_router)
|
||||
app.include_router(celery_router)
|
||||
app.include_router(cascade_router)
|
||||
|
||||
|
||||
|
||||
128
src/dashboard/routes/tasks_celery.py
Normal file
128
src/dashboard/routes/tasks_celery.py
Normal file
@@ -0,0 +1,128 @@
|
||||
"""Celery task queue routes — view and manage background tasks.
|
||||
|
||||
GET /celery — render the Celery task queue page
|
||||
GET /celery/api — JSON list of tasks
|
||||
POST /celery/api — submit a new background task
|
||||
GET /celery/api/{id} — get status of a specific task
|
||||
POST /celery/api/{id}/revoke — cancel a running task
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/celery", tags=["celery"])
|
||||
templates = Jinja2Templates(directory=str(Path(__file__).parent.parent / "templates"))
|
||||
|
||||
# In-memory record of submitted task IDs for the dashboard display.
|
||||
# In production this would use the Celery result backend directly,
|
||||
# but this lightweight list keeps the UI functional without Redis.
|
||||
_submitted_tasks: list[dict] = []
|
||||
_MAX_TASK_HISTORY = 100
|
||||
|
||||
|
||||
@router.get("", response_class=HTMLResponse)
|
||||
async def tasks_page(request: Request):
|
||||
"""Render the Celery task queue page."""
|
||||
from infrastructure.celery.app import celery_app
|
||||
|
||||
celery_available = celery_app is not None
|
||||
tasks = _get_tasks_with_status()
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"celery_tasks.html",
|
||||
{"tasks": tasks, "celery_available": celery_available},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api", response_class=JSONResponse)
|
||||
async def tasks_api():
|
||||
"""Return task list as JSON with current status."""
|
||||
return _get_tasks_with_status()
|
||||
|
||||
|
||||
@router.post("/api", response_class=JSONResponse)
|
||||
async def submit_task_api(request: Request):
|
||||
"""Submit a new background task.
|
||||
|
||||
Body: {"prompt": "...", "agent_id": "timmy"}
|
||||
"""
|
||||
from infrastructure.celery.client import submit_chat_task
|
||||
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception:
|
||||
return JSONResponse({"error": "Invalid JSON body"}, status_code=400)
|
||||
|
||||
prompt = body.get("prompt", "").strip()
|
||||
if not prompt:
|
||||
return JSONResponse({"error": "prompt is required"}, status_code=400)
|
||||
|
||||
agent_id = body.get("agent_id", "timmy")
|
||||
task_id = submit_chat_task(prompt=prompt, agent_id=agent_id)
|
||||
|
||||
if task_id is None:
|
||||
return JSONResponse(
|
||||
{"error": "Celery is not available. Start Redis and a Celery worker."},
|
||||
status_code=503,
|
||||
)
|
||||
|
||||
task_record = {
|
||||
"task_id": task_id,
|
||||
"prompt": prompt[:200],
|
||||
"agent_id": agent_id,
|
||||
"state": "PENDING",
|
||||
}
|
||||
_submitted_tasks.append(task_record)
|
||||
if len(_submitted_tasks) > _MAX_TASK_HISTORY:
|
||||
_submitted_tasks.pop(0)
|
||||
|
||||
return JSONResponse(task_record, status_code=202)
|
||||
|
||||
|
||||
@router.get("/api/{task_id}", response_class=JSONResponse)
|
||||
async def task_status_api(task_id: str):
|
||||
"""Get status of a specific task."""
|
||||
from infrastructure.celery.client import get_task_status
|
||||
|
||||
status = get_task_status(task_id)
|
||||
if status is None:
|
||||
return JSONResponse(
|
||||
{"error": "Celery is not available or task not found"},
|
||||
status_code=503,
|
||||
)
|
||||
return status
|
||||
|
||||
|
||||
@router.post("/api/{task_id}/revoke", response_class=JSONResponse)
|
||||
async def revoke_task_api(task_id: str):
|
||||
"""Cancel a pending or running task."""
|
||||
from infrastructure.celery.client import revoke_task
|
||||
|
||||
success = revoke_task(task_id)
|
||||
if not success:
|
||||
return JSONResponse(
|
||||
{"error": "Failed to revoke task (Celery unavailable)"},
|
||||
status_code=503,
|
||||
)
|
||||
return {"task_id": task_id, "status": "revoked"}
|
||||
|
||||
|
||||
def _get_tasks_with_status() -> list[dict]:
|
||||
"""Enrich submitted tasks with current Celery status."""
|
||||
from infrastructure.celery.client import get_task_status
|
||||
|
||||
enriched = []
|
||||
for record in reversed(_submitted_tasks):
|
||||
status = get_task_status(record["task_id"])
|
||||
if status:
|
||||
record_copy = {**record, **status}
|
||||
else:
|
||||
record_copy = {**record, "state": "UNKNOWN"}
|
||||
enriched.append(record_copy)
|
||||
return enriched
|
||||
221
src/dashboard/templates/celery_tasks.html
Normal file
221
src/dashboard/templates/celery_tasks.html
Normal file
@@ -0,0 +1,221 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Timmy Time — Background Tasks{% endblock %}
|
||||
|
||||
{% block extra_styles %}
|
||||
<style>
|
||||
.celery-container { max-width: 720px; }
|
||||
|
||||
.celery-header {
|
||||
border-left: 3px solid var(--green);
|
||||
padding-left: 1rem;
|
||||
}
|
||||
.celery-title {
|
||||
font-size: 1.6rem;
|
||||
font-weight: 700;
|
||||
color: var(--green);
|
||||
letter-spacing: 0.04em;
|
||||
font-family: var(--font);
|
||||
}
|
||||
.celery-subtitle {
|
||||
font-size: 0.75rem;
|
||||
color: var(--text-dim);
|
||||
margin-top: 0.25rem;
|
||||
}
|
||||
|
||||
.celery-card {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
padding: 1rem;
|
||||
margin-bottom: 0.75rem;
|
||||
background: rgba(24, 10, 45, 0.5);
|
||||
transition: border-color 0.2s;
|
||||
}
|
||||
.celery-card:hover {
|
||||
border-color: var(--green);
|
||||
}
|
||||
|
||||
.celery-prompt {
|
||||
font-size: 0.9rem;
|
||||
color: var(--text-bright);
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
line-height: 1.55;
|
||||
}
|
||||
|
||||
.celery-meta {
|
||||
display: flex;
|
||||
gap: 0.75rem;
|
||||
align-items: center;
|
||||
margin-bottom: 0.5rem;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.celery-id {
|
||||
font-size: 0.72rem;
|
||||
color: var(--text-dim);
|
||||
font-family: var(--font);
|
||||
}
|
||||
|
||||
.cstate-badge {
|
||||
font-size: 0.68rem;
|
||||
padding: 0.15em 0.5em;
|
||||
border-radius: 4px;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
}
|
||||
.cstate-PENDING { background: rgba(255, 193, 7, 0.15); color: var(--amber); }
|
||||
.cstate-STARTED { background: rgba(56, 189, 248, 0.15); color: #38bdf8; }
|
||||
.cstate-SUCCESS { background: rgba(0, 232, 122, 0.15); color: var(--green); }
|
||||
.cstate-FAILURE { background: rgba(239, 68, 68, 0.2); color: #f87171; }
|
||||
.cstate-REVOKED { background: rgba(148, 163, 184, 0.15); color: #94a3b8; }
|
||||
.cstate-UNKNOWN { background: rgba(148, 163, 184, 0.1); color: #64748b; }
|
||||
|
||||
.celery-result {
|
||||
font-size: 0.82rem;
|
||||
color: var(--text-dim);
|
||||
margin-top: 0.5rem;
|
||||
padding: 0.5rem;
|
||||
background: rgba(0, 0, 0, 0.25);
|
||||
border-radius: 4px;
|
||||
max-height: 150px;
|
||||
overflow-y: auto;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.celery-submit-form {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 1.5rem;
|
||||
}
|
||||
.celery-submit-form input {
|
||||
flex: 1;
|
||||
padding: 0.5rem 0.75rem;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius-md);
|
||||
background: rgba(24, 10, 45, 0.6);
|
||||
color: var(--text-bright);
|
||||
font-size: 0.85rem;
|
||||
font-family: var(--font);
|
||||
}
|
||||
.celery-submit-form input::placeholder { color: var(--text-dim); }
|
||||
.celery-submit-form button {
|
||||
padding: 0.5rem 1rem;
|
||||
border: 1px solid var(--green);
|
||||
border-radius: var(--radius-md);
|
||||
background: rgba(0, 232, 122, 0.1);
|
||||
color: var(--green);
|
||||
font-size: 0.82rem;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.celery-submit-form button:hover { background: rgba(0, 232, 122, 0.2); }
|
||||
|
||||
.celery-empty {
|
||||
text-align: center;
|
||||
color: var(--text-dim);
|
||||
padding: 3rem 0;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
.celery-offline {
|
||||
color: var(--amber);
|
||||
font-size: 0.82rem;
|
||||
padding: 0.5rem;
|
||||
border: 1px solid rgba(255, 193, 7, 0.3);
|
||||
border-radius: var(--radius-md);
|
||||
margin-bottom: 1rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
@media (max-width: 576px) {
|
||||
.celery-title { font-size: 1.3rem; }
|
||||
.celery-submit-form { flex-direction: column; }
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="container celery-container py-4">
|
||||
|
||||
<div class="celery-header mb-4">
|
||||
<div class="celery-title">Background Tasks</div>
|
||||
<div class="celery-subtitle">
|
||||
Tasks processed by Celery workers — submit work for Timmy to handle in the background.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if not celery_available %}
|
||||
<div class="celery-offline">
|
||||
Celery is offline. Start Redis and a Celery worker to enable background tasks.
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Submit form -->
|
||||
<form class="celery-submit-form" id="celery-form" onsubmit="return submitCeleryTask(event)">
|
||||
<input type="text" id="celery-prompt"
|
||||
placeholder="Describe a task for Timmy to work on in the background..." autocomplete="off">
|
||||
<button type="submit">Submit Task</button>
|
||||
</form>
|
||||
|
||||
<div class="card mc-panel">
|
||||
<div class="card-header mc-panel-header d-flex justify-content-between align-items-center">
|
||||
<span>// BACKGROUND TASKS</span>
|
||||
<span class="badge" style="background:rgba(0,232,122,0.15); color:var(--green);" id="celery-count">{{ tasks | length }} tasks</span>
|
||||
</div>
|
||||
<div class="card-body p-3" id="celery-task-list"
|
||||
hx-get="/celery/api"
|
||||
hx-trigger="every 3s"
|
||||
hx-swap="innerHTML">
|
||||
|
||||
{% if tasks %}
|
||||
{% for task in tasks %}
|
||||
<div class="celery-card">
|
||||
<div class="celery-meta">
|
||||
<span class="cstate-badge cstate-{{ task.state | default('UNKNOWN') }}">{{ task.state | default('UNKNOWN') }}</span>
|
||||
<span class="celery-id">{{ task.task_id[:12] }}...</span>
|
||||
<span class="celery-id">{{ task.agent_id | default('timmy') }}</span>
|
||||
</div>
|
||||
<div class="celery-prompt">{{ task.prompt | default('') | e }}</div>
|
||||
{% if task.result %}
|
||||
<div class="celery-result">{{ task.result | string | truncate(500) | e }}</div>
|
||||
{% endif %}
|
||||
{% if task.error %}
|
||||
<div class="celery-result" style="color: #f87171;">{{ task.error | e }}</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
<div class="celery-empty">
|
||||
No background tasks yet. Submit one above or ask Timmy to work on something.
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function submitCeleryTask(e) {
|
||||
e.preventDefault();
|
||||
var input = document.getElementById('celery-prompt');
|
||||
var prompt = input.value.trim();
|
||||
if (!prompt) return false;
|
||||
|
||||
fetch('/celery/api', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({prompt: prompt})
|
||||
}).then(function(r) { return r.json(); }).then(function() {
|
||||
input.value = '';
|
||||
htmx.trigger('#celery-task-list', 'htmx:load');
|
||||
}).catch(function(err) {
|
||||
console.error('Failed to submit task:', err);
|
||||
});
|
||||
return false;
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
22
src/infrastructure/celery/__init__.py
Normal file
22
src/infrastructure/celery/__init__.py
Normal file
@@ -0,0 +1,22 @@
|
||||
"""Celery task queue integration — optional background task processing.
|
||||
|
||||
Gracefully degrades when Redis or Celery are unavailable.
|
||||
"""
|
||||
|
||||
from infrastructure.celery.app import celery_app
|
||||
from infrastructure.celery.client import (
|
||||
get_active_tasks,
|
||||
get_task_status,
|
||||
revoke_task,
|
||||
submit_chat_task,
|
||||
submit_tool_task,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"celery_app",
|
||||
"get_active_tasks",
|
||||
"get_task_status",
|
||||
"revoke_task",
|
||||
"submit_chat_task",
|
||||
"submit_tool_task",
|
||||
]
|
||||
44
src/infrastructure/celery/app.py
Normal file
44
src/infrastructure/celery/app.py
Normal file
@@ -0,0 +1,44 @@
|
||||
"""Celery application factory with graceful degradation.
|
||||
|
||||
When Redis is unavailable or Celery is not installed, ``celery_app`` is set
|
||||
to ``None`` and all task submissions become safe no-ops.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
celery_app = None
|
||||
|
||||
_TEST_MODE = os.environ.get("TIMMY_TEST_MODE") == "1"
|
||||
|
||||
if not _TEST_MODE:
|
||||
try:
|
||||
from celery import Celery
|
||||
from config import settings
|
||||
|
||||
if not settings.celery_enabled:
|
||||
logger.info("Celery disabled via settings (celery_enabled=False)")
|
||||
else:
|
||||
celery_app = Celery("timmy")
|
||||
celery_app.conf.update(
|
||||
broker_url=settings.redis_url,
|
||||
result_backend=settings.redis_url,
|
||||
task_serializer="json",
|
||||
result_serializer="json",
|
||||
accept_content=["json"],
|
||||
result_expires=3600,
|
||||
task_track_started=True,
|
||||
worker_hijack_root_logger=False,
|
||||
)
|
||||
# Autodiscover tasks in the celery package
|
||||
celery_app.autodiscover_tasks(["infrastructure.celery"])
|
||||
logger.info("Celery app configured (broker=%s)", settings.redis_url)
|
||||
except ImportError:
|
||||
logger.info("Celery not installed — background tasks disabled")
|
||||
except Exception as exc:
|
||||
logger.warning("Celery setup failed (Redis down?): %s", exc)
|
||||
celery_app = None
|
||||
else:
|
||||
logger.debug("Celery disabled in test mode")
|
||||
150
src/infrastructure/celery/client.py
Normal file
150
src/infrastructure/celery/client.py
Normal file
@@ -0,0 +1,150 @@
|
||||
"""Client API for submitting and querying Celery tasks.
|
||||
|
||||
All functions gracefully return None/empty when Celery is unavailable.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_app():
|
||||
"""Get the Celery app instance."""
|
||||
from infrastructure.celery.app import celery_app
|
||||
return celery_app
|
||||
|
||||
|
||||
def submit_chat_task(
|
||||
prompt: str,
|
||||
agent_id: str = "timmy",
|
||||
session_id: str = "celery",
|
||||
) -> str | None:
|
||||
"""Submit a chat task to the Celery queue.
|
||||
|
||||
Returns:
|
||||
Task ID string, or None if Celery is unavailable.
|
||||
"""
|
||||
app = _get_app()
|
||||
if app is None:
|
||||
logger.debug("Celery unavailable — chat task not submitted")
|
||||
return None
|
||||
|
||||
try:
|
||||
from infrastructure.celery.tasks import run_agent_chat
|
||||
result = run_agent_chat.delay(prompt, agent_id=agent_id, session_id=session_id)
|
||||
logger.info("Submitted chat task %s for %s", result.id, agent_id)
|
||||
return result.id
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to submit chat task: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def submit_tool_task(
|
||||
tool_name: str,
|
||||
kwargs: dict | None = None,
|
||||
agent_id: str = "timmy",
|
||||
) -> str | None:
|
||||
"""Submit a tool execution task to the Celery queue.
|
||||
|
||||
Returns:
|
||||
Task ID string, or None if Celery is unavailable.
|
||||
"""
|
||||
app = _get_app()
|
||||
if app is None:
|
||||
logger.debug("Celery unavailable — tool task not submitted")
|
||||
return None
|
||||
|
||||
try:
|
||||
from infrastructure.celery.tasks import execute_tool
|
||||
result = execute_tool.delay(tool_name, kwargs=kwargs or {}, agent_id=agent_id)
|
||||
logger.info("Submitted tool task %s: %s", result.id, tool_name)
|
||||
return result.id
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to submit tool task: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def get_task_status(task_id: str) -> dict[str, Any] | None:
|
||||
"""Get status of a Celery task.
|
||||
|
||||
Returns:
|
||||
Dict with state, result, etc., or None if Celery unavailable.
|
||||
"""
|
||||
app = _get_app()
|
||||
if app is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
result = app.AsyncResult(task_id)
|
||||
data: dict[str, Any] = {
|
||||
"task_id": task_id,
|
||||
"state": result.state,
|
||||
"ready": result.ready(),
|
||||
}
|
||||
if result.ready():
|
||||
data["result"] = result.result
|
||||
if result.failed():
|
||||
data["error"] = str(result.result)
|
||||
return data
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to get task status: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def get_active_tasks() -> list[dict[str, Any]]:
|
||||
"""List currently active/reserved tasks.
|
||||
|
||||
Returns:
|
||||
List of task dicts, or empty list if Celery unavailable.
|
||||
"""
|
||||
app = _get_app()
|
||||
if app is None:
|
||||
return []
|
||||
|
||||
try:
|
||||
inspector = app.control.inspect()
|
||||
active = inspector.active() or {}
|
||||
reserved = inspector.reserved() or {}
|
||||
|
||||
tasks = []
|
||||
for worker_tasks in active.values():
|
||||
for t in worker_tasks:
|
||||
tasks.append({
|
||||
"task_id": t.get("id"),
|
||||
"name": t.get("name"),
|
||||
"state": "STARTED",
|
||||
"args": t.get("args"),
|
||||
"worker": t.get("hostname"),
|
||||
})
|
||||
for worker_tasks in reserved.values():
|
||||
for t in worker_tasks:
|
||||
tasks.append({
|
||||
"task_id": t.get("id"),
|
||||
"name": t.get("name"),
|
||||
"state": "PENDING",
|
||||
"args": t.get("args"),
|
||||
})
|
||||
return tasks
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to inspect active tasks: %s", exc)
|
||||
return []
|
||||
|
||||
|
||||
def revoke_task(task_id: str) -> bool:
|
||||
"""Revoke (cancel) a pending or running task.
|
||||
|
||||
Returns:
|
||||
True if revoke was sent, False if Celery unavailable.
|
||||
"""
|
||||
app = _get_app()
|
||||
if app is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
app.control.revoke(task_id, terminate=True)
|
||||
logger.info("Revoked task %s", task_id)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to revoke task: %s", exc)
|
||||
return False
|
||||
147
src/infrastructure/celery/tasks.py
Normal file
147
src/infrastructure/celery/tasks.py
Normal file
@@ -0,0 +1,147 @@
|
||||
"""Celery task definitions for background processing.
|
||||
|
||||
Tasks:
|
||||
- run_agent_chat: Execute a chat prompt via Timmy's session
|
||||
- execute_tool: Run a specific tool function asynchronously
|
||||
- run_thinking_cycle: Execute one thinking engine cycle
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_app():
|
||||
"""Get the Celery app (lazy import to avoid circular deps)."""
|
||||
from infrastructure.celery.app import celery_app
|
||||
return celery_app
|
||||
|
||||
|
||||
_app = _get_app()
|
||||
|
||||
if _app is not None:
|
||||
|
||||
@_app.task(bind=True, name="infrastructure.celery.tasks.run_agent_chat")
|
||||
def run_agent_chat(self, prompt, agent_id="timmy", session_id="celery"):
|
||||
"""Execute a chat prompt against Timmy's agent session.
|
||||
|
||||
Args:
|
||||
prompt: The message to send to the agent.
|
||||
agent_id: Agent identifier (currently only "timmy" supported).
|
||||
session_id: Chat session ID for context continuity.
|
||||
|
||||
Returns:
|
||||
Dict with agent_id, response, and completed_at.
|
||||
"""
|
||||
logger.info("Celery task [%s]: chat prompt for %s", self.request.id, agent_id)
|
||||
try:
|
||||
from timmy.session import chat
|
||||
response = chat(prompt, session_id=session_id)
|
||||
result = {
|
||||
"agent_id": agent_id,
|
||||
"prompt": prompt[:200],
|
||||
"response": response,
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
_log_completion("run_agent_chat", self.request.id, success=True)
|
||||
return result
|
||||
except Exception as exc:
|
||||
logger.error("Celery chat task failed: %s", exc)
|
||||
_log_completion("run_agent_chat", self.request.id, success=False)
|
||||
return {
|
||||
"agent_id": agent_id,
|
||||
"prompt": prompt[:200],
|
||||
"error": str(exc),
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
@_app.task(bind=True, name="infrastructure.celery.tasks.execute_tool")
|
||||
def execute_tool(self, tool_name, kwargs=None, agent_id="timmy"):
|
||||
"""Run a specific tool function asynchronously.
|
||||
|
||||
Args:
|
||||
tool_name: Name of the tool to execute (e.g., "web_search").
|
||||
kwargs: Dict of keyword arguments for the tool.
|
||||
agent_id: Agent requesting the tool execution.
|
||||
|
||||
Returns:
|
||||
Dict with tool_name, result, and success flag.
|
||||
"""
|
||||
kwargs = kwargs or {}
|
||||
logger.info("Celery task [%s]: tool=%s for %s", self.request.id, tool_name, agent_id)
|
||||
try:
|
||||
from timmy.tools import create_full_toolkit
|
||||
toolkit = create_full_toolkit()
|
||||
if toolkit is None:
|
||||
return {"tool_name": tool_name, "error": "Toolkit unavailable", "success": False}
|
||||
|
||||
# Find and call the tool function
|
||||
tool_fn = None
|
||||
for fn in toolkit.functions.values():
|
||||
if fn.name == tool_name:
|
||||
tool_fn = fn
|
||||
break
|
||||
|
||||
if tool_fn is None:
|
||||
return {"tool_name": tool_name, "error": f"Tool '{tool_name}' not found", "success": False}
|
||||
|
||||
result = tool_fn.entrypoint(**kwargs)
|
||||
_log_completion("execute_tool", self.request.id, success=True)
|
||||
return {
|
||||
"tool_name": tool_name,
|
||||
"result": str(result)[:5000],
|
||||
"success": True,
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.error("Celery tool task failed: %s", exc)
|
||||
_log_completion("execute_tool", self.request.id, success=False)
|
||||
return {"tool_name": tool_name, "error": str(exc), "success": False}
|
||||
|
||||
@_app.task(bind=True, name="infrastructure.celery.tasks.run_thinking_cycle")
|
||||
def run_thinking_cycle(self):
|
||||
"""Execute one thinking engine cycle in the background.
|
||||
|
||||
Returns:
|
||||
Dict with thought data or None if thinking is disabled.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
logger.info("Celery task [%s]: thinking cycle", self.request.id)
|
||||
try:
|
||||
from timmy.thinking import thinking_engine
|
||||
|
||||
# Run the async think_once in a sync context
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
thought = loop.run_until_complete(thinking_engine.think_once())
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
if thought:
|
||||
_log_completion("run_thinking_cycle", self.request.id, success=True)
|
||||
return {
|
||||
"thought_id": thought.id,
|
||||
"content": thought.content,
|
||||
"seed_type": thought.seed_type,
|
||||
"created_at": thought.created_at,
|
||||
}
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.error("Celery thinking task failed: %s", exc)
|
||||
_log_completion("run_thinking_cycle", self.request.id, success=False)
|
||||
return None
|
||||
|
||||
|
||||
def _log_completion(task_name, task_id, success=True):
|
||||
"""Log task completion to the Spark engine if available."""
|
||||
try:
|
||||
from spark.engine import spark_engine
|
||||
spark_engine.on_tool_executed(
|
||||
agent_id="celery-worker",
|
||||
tool_name=f"celery.{task_name}",
|
||||
success=success,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -476,6 +476,14 @@ def create_full_toolkit(base_dir: str | Path | None = None):
|
||||
except Exception:
|
||||
logger.debug("Delegation tools not available")
|
||||
|
||||
# Background task submission via Celery
|
||||
try:
|
||||
from timmy.tools_celery import submit_background_task
|
||||
|
||||
toolkit.register(submit_background_task, name="submit_background_task")
|
||||
except Exception:
|
||||
logger.debug("Background task tool not available")
|
||||
|
||||
return toolkit
|
||||
|
||||
|
||||
@@ -596,6 +604,11 @@ def get_all_available_tools() -> dict[str, dict]:
|
||||
"description": "Local AI coding assistant using Ollama (qwen2.5:14b or deepseek-coder)",
|
||||
"available_in": ["forge", "orchestrator"],
|
||||
},
|
||||
"submit_background_task": {
|
||||
"name": "Background Task",
|
||||
"description": "Submit a task to the Celery background queue for async processing",
|
||||
"available_in": ["orchestrator"],
|
||||
},
|
||||
}
|
||||
|
||||
# ── Git tools ─────────────────────────────────────────────────────────────
|
||||
|
||||
60
src/timmy/tools_celery.py
Normal file
60
src/timmy/tools_celery.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""Celery background task tool — allows Timmy to submit tasks to a worker queue.
|
||||
|
||||
When Celery/Redis is unavailable, returns a graceful error dict.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def submit_background_task(task_description: str, agent_id: str = "timmy") -> dict[str, Any]:
|
||||
"""Submit a task to run in the background via Celery.
|
||||
|
||||
Use this tool when a user asks you to work on something that might
|
||||
take a while — research, analysis, code generation, etc. The task
|
||||
will be processed by a background worker and the user can check
|
||||
progress on the /tasks page.
|
||||
|
||||
Args:
|
||||
task_description: What to work on (sent as a chat prompt to the agent).
|
||||
agent_id: Which agent should handle it (default: timmy).
|
||||
|
||||
Returns:
|
||||
Dict with task_id, status, and message.
|
||||
"""
|
||||
try:
|
||||
from infrastructure.celery.client import submit_chat_task
|
||||
|
||||
task_id = submit_chat_task(
|
||||
prompt=task_description,
|
||||
agent_id=agent_id,
|
||||
session_id="celery-background",
|
||||
)
|
||||
|
||||
if task_id is None:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Background task queue is not available (Redis/Celery not running).",
|
||||
"task_id": None,
|
||||
}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"task_id": task_id,
|
||||
"agent_id": agent_id,
|
||||
"status": "submitted",
|
||||
"message": (
|
||||
f"Background task submitted (ID: {task_id[:8]}...). "
|
||||
f"Check progress at /tasks. Task: {task_description[:100]}"
|
||||
),
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Failed to submit background task: %s", exc)
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(exc),
|
||||
"task_id": None,
|
||||
}
|
||||
@@ -33,6 +33,9 @@ for _mod in [
|
||||
"pyzbar",
|
||||
"pyzbar.pyzbar",
|
||||
"requests",
|
||||
"celery",
|
||||
"celery.app",
|
||||
"celery.result",
|
||||
]:
|
||||
sys.modules.setdefault(_mod, MagicMock())
|
||||
|
||||
|
||||
94
tests/test_celery_integration.py
Normal file
94
tests/test_celery_integration.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""Tests for Celery integration — graceful degradation when Celery is unavailable."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestCeleryClient:
|
||||
"""Test the Celery client API gracefully degrades in test mode."""
|
||||
|
||||
def test_submit_chat_task_returns_none_when_unavailable(self):
|
||||
from infrastructure.celery.client import submit_chat_task
|
||||
|
||||
result = submit_chat_task("test prompt")
|
||||
assert result is None
|
||||
|
||||
def test_submit_tool_task_returns_none_when_unavailable(self):
|
||||
from infrastructure.celery.client import submit_tool_task
|
||||
|
||||
result = submit_tool_task("web_search", {"query": "test"})
|
||||
assert result is None
|
||||
|
||||
def test_get_task_status_returns_none_when_unavailable(self):
|
||||
from infrastructure.celery.client import get_task_status
|
||||
|
||||
result = get_task_status("fake-task-id")
|
||||
assert result is None
|
||||
|
||||
def test_get_active_tasks_returns_empty_when_unavailable(self):
|
||||
from infrastructure.celery.client import get_active_tasks
|
||||
|
||||
result = get_active_tasks()
|
||||
assert result == []
|
||||
|
||||
def test_revoke_task_returns_false_when_unavailable(self):
|
||||
from infrastructure.celery.client import revoke_task
|
||||
|
||||
result = revoke_task("fake-task-id")
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestCeleryApp:
|
||||
"""Test the Celery app is None in test mode."""
|
||||
|
||||
def test_celery_app_is_none_in_test_mode(self):
|
||||
from infrastructure.celery.app import celery_app
|
||||
|
||||
assert celery_app is None
|
||||
|
||||
|
||||
class TestBackgroundTaskTool:
|
||||
"""Test the submit_background_task agent tool."""
|
||||
|
||||
def test_returns_failure_when_celery_unavailable(self):
|
||||
from timmy.tools_celery import submit_background_task
|
||||
|
||||
result = submit_background_task("research something")
|
||||
assert result["success"] is False
|
||||
assert result["task_id"] is None
|
||||
assert "not available" in result["error"]
|
||||
|
||||
|
||||
class TestCeleryRoutes:
|
||||
"""Test the Celery dashboard routes."""
|
||||
|
||||
def test_celery_page_renders(self, client):
|
||||
response = client.get("/celery")
|
||||
assert response.status_code == 200
|
||||
assert "Background Tasks" in response.text
|
||||
|
||||
def test_celery_api_returns_empty_list(self, client):
|
||||
response = client.get("/celery/api")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == []
|
||||
|
||||
def test_celery_submit_requires_prompt(self, client):
|
||||
response = client.post(
|
||||
"/celery/api",
|
||||
json={"agent_id": "timmy"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_celery_submit_returns_503_when_unavailable(self, client):
|
||||
response = client.post(
|
||||
"/celery/api",
|
||||
json={"prompt": "do something", "agent_id": "timmy"},
|
||||
)
|
||||
assert response.status_code == 503
|
||||
|
||||
def test_celery_task_status_returns_503_when_unavailable(self, client):
|
||||
response = client.get("/celery/api/fake-id")
|
||||
assert response.status_code == 503
|
||||
|
||||
def test_celery_revoke_returns_503_when_unavailable(self, client):
|
||||
response = client.post("/celery/api/fake-id/revoke")
|
||||
assert response.status_code == 503
|
||||
Reference in New Issue
Block a user