首先应先安装各种编译需要的包:
[root@bogon upir]# dnf install libxslt-devel redhat-rpm-config python-devel openssl-devel
再进行Scrapy安装:
[root@bogon upir]# pip install Scrapy
Collecting Scrapy
Downloading Scrapy-1.1.1-py2.py3-none-any.whl (295kB)
100% |████████████████████████████████| 296kB 40kB/s
Collecting lxml (from Scrapy)
Downloading lxml-3.6.0.tar.gz (3.7MB)
100% |████████████████████████████████| 3.7MB 47kB/s
Collecting w3lib>=1.14.2 (from Scrapy)
Downloading w3lib-1.14.3-py2.py3-none-any.whl
Collecting service-identity (from Scrapy)
Downloading service_identity-16.0.0-py2.py3-none-any.whl
Collecting pyOpenSSL (from Scrapy)
Downloading pyOpenSSL-16.0.0-py2.py3-none-any.whl (45kB)
100% |████████████████████████████████| 51kB 40kB/s
Collecting six>=1.5.2 (from Scrapy)
Downloading six-1.10.0-py2.py3-none-any.whl
Collecting parsel>=0.9.3 (from Scrapy)
Downloading parsel-1.0.2-py2.py3-none-any.whl
Collecting PyDispatcher>=2.0.5 (from Scrapy)
Downloading PyDispatcher-2.0.5.tar.gz
Collecting queuelib (from Scrapy)
Downloading queuelib-1.4.2-py2.py3-none-any.whl
Collecting cssselect>=0.9 (from Scrapy)
Downloading cssselect-0.9.2-py2.py3-none-any.whl
Collecting Twisted>=10.0.0 (from Scrapy)
Downloading Twisted-16.3.0.tar.bz2 (2.9MB)
100% |████████████████████████████████| 2.9MB 33kB/s
Collecting pyasn1-modules (from service-identity->Scrapy)
Downloading pyasn1_modules-0.0.8-py2.py3-none-any.whl
Collecting pyasn1 (from service-identity->Scrapy)
Downloading pyasn1-0.1.9-py2.py3-none-any.whl
Collecting attrs (from service-identity->Scrapy)
Downloading attrs-16.0.0-py2.py3-none-any.whl
Collecting cryptography>=1.3 (from pyOpenSSL->Scrapy)
Downloading cryptography-1.4.tar.gz (399kB)
100% |████████████████████████████████| 409kB 48kB/s
Collecting zope.interface>=3.6.0 (from Twisted>=10.0.0->Scrapy)
Downloading zope.interface-4.2.0.tar.gz (146kB)
100% |████████████████████████████████| 153kB 56kB/s
Collecting idna>=2.0 (from cryptography>=1.3->pyOpenSSL->Scrapy)
Downloading idna-2.1-py2.py3-none-any.whl (54kB)
100% |████████████████████████████████| 61kB 60kB/s
Requirement already satisfied (use --upgrade to upgrade): setuptools>=11.3 in /usr/lib/python2.7/site-packages (from cryptography>=1.3->pyOpenSSL->Scrapy)
Collecting enum34 (from cryptography>=1.3->pyOpenSSL->Scrapy)
Downloading enum34-1.1.6-py2-none-any.whl
Collecting ipaddress (from cryptography>=1.3->pyOpenSSL->Scrapy)
Downloading ipaddress-1.0.16-py27-none-any.whl
Collecting cffi>=1.4.1 (from cryptography>=1.3->pyOpenSSL->Scrapy)
Downloading cffi-1.7.0-cp27-cp27mu-manylinux1_x86_64.whl (383kB)
100% |████████████████████████████████| 389kB 58kB/s
Collecting pycparser (from cffi>=1.4.1->cryptography>=1.3->pyOpenSSL->Scrapy)
Downloading pycparser-2.14.tar.gz (223kB)
100% |████████████████████████████████| 225kB 57kB/s
Installing collected packages: lxml, six, w3lib, pyasn1, pyasn1-modules, idna, enum34, ipaddress, pycparser, cffi, cryptography, pyOpenSSL, attrs, service-identity, cssselect, parsel, PyDispatcher, queuelib, zope.interface, Twisted, Scrapy
Running setup.py install for lxml ... error
Complete output from command /usr/bin/python -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-7xYR1h/lxml/setup.py';exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))" install --record /tmp/pip-h5ATXl-record/install-record.txt --single-version-externally-managed --compile:
Building lxml version 3.6.0.
Building without Cython.
ERROR: /bin/sh: xslt-config: 未找到命令
** make sure the development packages of libxml2 and libxslt are installed **
Using build configuration of libxslt
running install
running build
running build_py
creating build
creating build/lib.linux-x86_64-2.7
creating build/lib.linux-x86_64-2.7/lxml
copying src/lxml/doctestcompare.py -> build/lib.linux-x86_64-2.7/lxml
copying src/lxml/__init__.py -> build/lib.linux-x86_64-2.7/lxml
copying src/lxml/_elementpath.py -> build/lib.linux-x86_64-2.7/lxml
copying src/lxml/sax.py -> build/lib.linux-x86_64-2.7/lxml
copying src/lxml/ElementInclude.py -> build/lib.linux-x86_64-2.7/lxml
copying src/lxml/usedoctest.py -> build/lib.linux-x86_64-2.7/lxml
copying src/lxml/pyclasslookup.py -> build/lib.linux-x86_64-2.7/lxml
copying src/lxml/cssselect.py -> build/lib.linux-x86_64-2.7/lxml
copying src/lxml/builder.py -> build/lib.linux-x86_64-2.7/lxml
creating build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/__init__.py -> build/lib.linux-x86_64-2.7/lxml/includes
creating build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/diff.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/defs.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/__init__.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/ElementSoup.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/soupparser.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/_setmixin.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/usedoctest.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/formfill.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/clean.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/_diffcommand.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/html5parser.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/builder.py -> build/lib.linux-x86_64-2.7/lxml/html
copying src/lxml/html/_html5builder.py -> build/lib.linux-x86_64-2.7/lxml/html
creating build/lib.linux-x86_64-2.7/lxml/isoschematron
copying src/lxml/isoschematron/__init__.py -> build/lib.linux-x86_64-2.7/lxml/isoschematron
copying src/lxml/lxml.etree.h -> build/lib.linux-x86_64-2.7/lxml
copying src/lxml/lxml.etree_api.h -> build/lib.linux-x86_64-2.7/lxml
copying src/lxml/includes/xinclude.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/uri.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/xmlschema.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/htmlparser.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/xslt.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/tree.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/xmlparser.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/xpath.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/schematron.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/config.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/dtdvalid.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/c14n.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/etreepublic.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/xmlerror.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/relaxng.pxd -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/etree_defs.h -> build/lib.linux-x86_64-2.7/lxml/includes
copying src/lxml/includes/lxml-version.h -> build/lib.linux-x86_64-2.7/lxml/includes
creating build/lib.linux-x86_64-2.7/lxml/isoschematron/resources
creating build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/rng
copying src/lxml/isoschematron/resources/rng/iso-schematron.rng -> build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/rng
creating build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/xsl
copying src/lxml/isoschematron/resources/xsl/RNG2Schtrn.xsl -> build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/xsl
copying src/lxml/isoschematron/resources/xsl/XSD2Schtrn.xsl -> build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/xsl
creating build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/xsl/iso-schematron-xslt1
copying src/lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_schematron_skeleton_for_xslt1.xsl -> build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/xsl/iso-schematron-xslt1
copying src/lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_abstract_expand.xsl -> build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/xsl/iso-schematron-xslt1
copying src/lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_svrl_for_xslt1.xsl -> build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/xsl/iso-schematron-xslt1
copying src/lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_schematron_message.xsl -> build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/xsl/iso-schematron-xslt1
copying src/lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_dsdl_include.xsl -> build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/xsl/iso-schematron-xslt1
copying src/lxml/isoschematron/resources/xsl/iso-schematron-xslt1/readme.txt -> build/lib.linux-x86_64-2.7/lxml/isoschematron/resources/xsl/iso-schematron-xslt1
running build_ext
building 'lxml.etree' extension
creating build/temp.linux-x86_64-2.7
creating build/temp.linux-x86_64-2.7/src
creating build/temp.linux-x86_64-2.7/src/lxml
gcc -pthread -fno-strict-aliasing -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -DNDEBUG -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong --param=ssp-buffer-size=4 -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -m64 -mtune=generic -D_GNU_SOURCE -fPIC -fwrapv -fPIC -Isrc/lxml/includes -I/usr/include/python2.7 -c src/lxml/lxml.etree.c -o build/temp.linux-x86_64-2.7/src/lxml/lxml.etree.o -w
gcc: 错误:/usr/lib/rpm/redhat/redhat-hardened-cc1:No such file or directory
Compile failed: command 'gcc' failed with exit status 1
creating tmp
cc -I/usr/include/libxml2 -c /tmp/xmlXPathInitxsOje_.c -o tmp/xmlXPathInitxsOje_.o
/tmp/xmlXPathInitxsOje_.c:1:26: 致命错误:libxml/xpath.h:No such file or directory
#include "libxml/xpath.h"
^
编译中断。
*********************************************************************************
Could not find function xmlCheckVersion in library libxml2. Is libxml2 installed?
*********************************************************************************
error: command 'gcc' failed with exit status 1
----------------------------------------
Command "/usr/bin/python -u -c "import setuptools, tokenize;__file__='/tmp/pip-build-7xYR1h/lxml/setup.py';exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\r\n', '\n'), __file__, 'exec'))" install --record /tmp/pip-h5ATXl-record/install-record.txt --single-version-externally-managed --compile" failed with error code 1 in /tmp/pip-build-7xYR1h/lxml/
上面是首次安装出现的一个问题,
下面是成功安装的:
[root@bogon upir]# pip install Scrapy
Collecting Scrapy
Using cached Scrapy-1.1.1-py2.py3-none-any.whl
Requirement already satisfied (use --upgrade to upgrade): lxml in /usr/lib64/python2.7/site-packages (from Scrapy)
Requirement already satisfied (use --upgrade to upgrade): w3lib>=1.14.2 in /usr/lib/python2.7/site-packages (from Scrapy)
Collecting service-identity (from Scrapy)
Using cached service_identity-16.0.0-py2.py3-none-any.whl
Collecting pyOpenSSL (from Scrapy)
Using cached pyOpenSSL-16.0.0-py2.py3-none-any.whl
Collecting parsel>=0.9.3 (from Scrapy)
Using cached parsel-1.0.2-py2.py3-none-any.whl
Collecting PyDispatcher>=2.0.5 (from Scrapy)
Using cached PyDispatcher-2.0.5.tar.gz
Requirement already satisfied (use --upgrade to upgrade): six>=1.5.2 in /usr/lib/python2.7/site-packages (from Scrapy)
Collecting queuelib (from Scrapy)
Using cached queuelib-1.4.2-py2.py3-none-any.whl
Collecting cssselect>=0.9 (from Scrapy)
Using cached cssselect-0.9.2-py2.py3-none-any.whl
Collecting Twisted>=10.0.0 (from Scrapy)
Using cached Twisted-16.3.0.tar.bz2
Requirement already satisfied (use --upgrade to upgrade): pyasn1-modules in /usr/lib/python2.7/site-packages (from service-identity->Scrapy)
Requirement already satisfied (use --upgrade to upgrade): pyasn1 in /usr/lib/python2.7/site-packages (from service-identity->Scrapy)
Collecting attrs (from service-identity->Scrapy)
Using cached attrs-16.0.0-py2.py3-none-any.whl
Collecting cryptography>=1.3 (from pyOpenSSL->Scrapy)
Using cached cryptography-1.4.tar.gz
Collecting zope.interface>=3.6.0 (from Twisted>=10.0.0->Scrapy)
Using cached zope.interface-4.2.0.tar.gz
Requirement already satisfied (use --upgrade to upgrade): idna>=2.0 in /usr/lib/python2.7/site-packages (from cryptography>=1.3->pyOpenSSL->Scrapy)
Requirement already satisfied (use --upgrade to upgrade): setuptools>=11.3 in /usr/lib/python2.7/site-packages (from cryptography>=1.3->pyOpenSSL->Scrapy)
Requirement already satisfied (use --upgrade to upgrade): enum34 in /usr/lib/python2.7/site-packages (from cryptography>=1.3->pyOpenSSL->Scrapy)
Requirement already satisfied (use --upgrade to upgrade): ipaddress in /usr/lib/python2.7/site-packages (from cryptography>=1.3->pyOpenSSL->Scrapy)
Requirement already satisfied (use --upgrade to upgrade): cffi>=1.4.1 in /usr/lib64/python2.7/site-packages (from cryptography>=1.3->pyOpenSSL->Scrapy)
Requirement already satisfied (use --upgrade to upgrade): pycparser in /usr/lib/python2.7/site-packages (from cffi>=1.4.1->cryptography>=1.3->pyOpenSSL->Scrapy)
Installing collected packages: cryptography, pyOpenSSL, attrs, service-identity, cssselect, parsel, PyDispatcher, queuelib, zope.interface, Twisted, Scrapy
Running setup.py install for cryptography ... done
Running setup.py install for PyDispatcher ... done
Running setup.py install for zope.interface ... done
Running setup.py install for Twisted ... done
Successfully installed PyDispatcher-2.0.5 Scrapy-1.1.1 Twisted-16.3.0 attrs-16.0.0 cryptography-1.4 cssselect-0.9.2 parsel-1.0.2 pyOpenSSL-16.0.0 queuelib-1.4.2 service-identity-16.0.0 zope.interface-4.2.0