Docker Toolbox
1.在Windows命令行执行docker-machine ssh [machine-name]进入VM bash
2.sudo vi /var/lib/boot2docker/profile
3.在--label provider=virtualbox的下一行添加--registry-mirror https://xxxxxxxx.mirror.aliyuncs.com
（阿里云 - 容器Hub服务控制台：https://cr.console.aliyun.com/ 可以得到一个专属的镜像加速地址，类似于“https://1234abcd.mirror.aliyuncs.com”。）
4.重启docker服务：sudo /etc/init.d/docker restart或者重启VM：exit退出VM bash，在Windows命令行中执行docker-machine restart
---------------------
作者：王旦东
来源：CSDN
原文：https://blog.csdn.net/wangdandong/article/details/68958210
版权声明：本文为博主原创文章，转载请附上博文链接！
"""
This modules implements the CrawlSpider which is the recommended spider to use
for scraping typical web sites that requires crawling pages.
See documentation in docs/topics/spiders.rst
"""
import copy
import six
from scrapy.httpimport Request, HtmlResponse
from scrapy.utils.spiderimport iterate_spider_output
from scrapy.spidersimport Spider
def identity(x):
return x
class Rule(object):
def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity):
self.link_extractor = link_extractor
self.callback = callback
self.cb_kwargs = cb_kwargsor {}
self.process_links = process_links
self.process_request = process_request
if followis None:
self.follow =False if callbackelse True
else:
self.follow = follow
class CrawlSpider(Spider):
rules = ()
def __init__(self, *a, **kw):
super(CrawlSpider,self).__init__(*a, **kw)
self._compile_rules()
def parse(self, response):
return self._parse_response(response,self.parse_start_url,cb_kwargs={},follow=True)
def parse_start_url(self, response):
return []
def process_results(self, response, results):
return results
def _build_request(self, rule, link):
r = Request(url=link.url,callback=self._response_downloaded)
r.meta.update(rule=rule,link_text=link.text)
return r
def _requests_to_follow(self, response):
if not isinstance(response, HtmlResponse):
return
seen =set()
for n, rulein enumerate(self._rules):
links = [lnkfor lnkin rule.link_extractor.extract_links(response)
if lnknot in seen]
if linksand rule.process_links:
links = rule.process_links(links)
for linkin links:
seen.add(link)
r =self._build_request(n, link)
yield rule.process_request(r)
def _response_downloaded(self, response):
rule =self._rules[response.meta['rule']]
return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)
def _parse_response(self, response, callback, cb_kwargs, follow=True):
if callback:
cb_res = callback(response, **cb_kwargs)or ()
cb_res =self.process_results(response, cb_res)
for requests_or_itemin iterate_spider_output(cb_res):
yield requests_or_item
if followand self._follow_links:
for request_or_itemin self._requests_to_follow(response):
yield request_or_item
def _compile_rules(self):
def get_method(method):
if callable(method):
return method
elif isinstance(method, six.string_types):
return getattr(self, method,None)
self._rules = [copy.copy(r)for rin self.rules]
for rulein self._rules:
rule.callback = get_method(rule.callback)
rule.process_links = get_method(rule.process_links)
rule.process_request = get_method(rule.process_request)
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider =super(CrawlSpider,cls).from_crawler(crawler, *args, **kwargs)
spider._follow_links = crawler.settings.getbool(
'CRAWLSPIDER_FOLLOW_LINKS',True)
return spider
def set_crawler(self, crawler):
super(CrawlSpider,self).set_crawler(crawler)
self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS',True)
