Dockerfile for scrapyd

最新推荐文章于 2025-08-05 15:25:51 发布

原创最新推荐文章于 2025-08-05 15:25:51 发布 · 360 阅读

0 ·

CC 4.0 BY-SA版权

kubernetes 同时被 2 个专栏收录

30 篇文章

订阅专栏

k8s

12 篇文章

订阅专栏

本文详细介绍如何使用Dockerfile构建Scrapyd容器，包括安装依赖、配置服务及暴露端口等步骤，为爬虫任务调度提供高效稳定的运行环境。

Dockerfile for scrapyd

FROM alpine:latest

RUN set -xe
&& apk update && apk add --no-cache python3 python3-dev
&& wget -O - https://bootstrap.pypa.io/get-pip.py | python3
&& apk add --no-cache gcc libc-dev libffi-dev libxslt-dev openssl-dev
&& pip install scrapy scrapyd scrapy-splash
&& mkdir /etc/scrapyd && cat < /etc/scrapyd/scrapyd.conf
[scrapyd]
http_port = 6800
bind_address = 0.0.0.0

debug             = off
logs_dir          = /var/log/scrapyd

max_proc          = 0
max_proc_per_cpu  = 4
poll_interval     = 5
jobs_to_keep      = 5
finished_to_keep  = 100

dbs_dir           = /var/lib/scrapyd/dbs
eggs_dir          = /var/lib/scrapyd/eggs
items_dir         = /var/lib/scrapyd/items

runner            = scrapyd.runner
application       = scrapyd.app.application
launcher          = scrapyd.launcher.Launcher

[services]
schedule.json     = scrapyd.webservice.Schedule
cancel.json       = scrapyd.webservice.Cancel
addversion.json   = scrapyd.webservice.AddVersion
listprojects.json = scrapyd.webservice.ListProjects
listversions.json = scrapyd.webservice.ListVersions
listspiders.json  = scrapyd.webservice.ListSpiders
delproject.json   = scrapyd.webservice.DeleteProject
delversion.json   = scrapyd.webservice.DeleteVersion
listjobs.json     = scrapyd.webservice.ListJobs
daemonstatus.json = scrapyd.webservice.DaemonStatus
EOF

EXPOSE 6800/tcp
CMD [“scrapyd”]