linw1995

def _process_queue(self, spider, slot):
    if slot.latercall and slot.latercall.active():
        # 由于同个 slot 会被 _process_queue 重复调用多次，情况 1，2
        # 当情况 3 在处理时，则不执行
        return


    # Delay queue processing if a download_delay is configured
    now = time()
    # 获取下载延迟，若是启用了 randomize_delay 则每次会获取到不同的下载延迟
    delay = slot.download_delay()
    if delay:
        # slot.lastseen 记录了最后一个 request 进行网络处理的时间
        penalty = delay - now + slot.lastseen
        if penalty > 0:
            # 当请求间隔不满足配置好的 DOWNLOAD_DELAY 时，则稍后再作处理
            slot.latercall = reactor.callLater(penalty, self._process_queue, spider, slot)
            return


    # Process enqueued requests if there are free slots to transfer for this slot
    while slot.queue and slot.free_transfer_slots() > 0:
        # 当实际并发数未达到限制时，则 request 进行网络处理
        slot.lastseen = now
        request, deferred = slot.queue.popleft()
        dfd = self._download(slot, request, spider)
        dfd.chainDeferred(deferred)
        # prevent burst if inter-request delays were configured
        if delay:
            # 当 DONWLOAD_DELAY > 0 时，要通过 _process_queue 进行延迟处理
            self._process_queue(spider, slot)
            break

        # 当 DOWNLOAD_DELAY == 0 时
        # 则一直进行 request 的网络处理，直到实际并发数达到限制，或者队列中没有新的 request

crawler.signals.connect(self._response_downloaded, signal=signals.response_downloaded)

def _response_downloaded(self, response, request, spider):
    key, slot = self._get_slot(request, spider)
    latency = request.meta.get('download_latency')
    # 获取 request 网络处理的耗时
    if latency is None or slot is None:
        return


    olddelay = slot.delay
    self._adjust_delay(slot, latency, response)
    # 调整 request 间的时间间隔
    if self.debug:
        ...

def _adjust_delay(self, slot, latency, response):
    """Define delay adjustment policy"""


    # If a server needs `latency` seconds to respond then
    # we should send a request each `latency/N` seconds
    # to have N requests processed in parallel
    # 如果服务器需要 latency 秒来响应，
    # 则我们需要以 latency / N 秒的时间间隔
    # 来并发地处理 N 个 requests
    target_delay = latency / self.target_concurrency


    # Adjust the delay to make it closer to target_delay
    # 调整 delay 使其逼近目标延迟时间 target_delay
    new_delay = (slot.delay + target_delay) / 2.0


    # If target delay is bigger than old delay, then use it instead of mean.
    # It works better with problematic sites.
    # 如果目标延迟时间 target_delay 大于旧的，则直接使用 target_delay
    # 这样的话，能更好的处理一些有问题的网站
    new_delay = max(target_delay, new_delay)


    # Make sure self.mindelay <= new_delay <= self.max_delay
    # 保证新的延迟 new_delay 大于最小延迟，小于最大延迟限制
    new_delay = min(max(self.mindelay, new_delay), self.maxdelay)


    # Dont adjust delay if response status != 200 and new delay is smaller
    # than old one, as error pages (and redirections) are usually small and
    # so tend to reduce latency, thus provoking a positive feedback by
    # reducing delay instead of increase.
    # 当响应的状态不为 200，及新的延迟小于旧的时，则不调整延迟。
    # 这是因为错误页或者重定向的响应时间小，会趋向于降低延迟
    # 但应该做的是增加延迟（错误页响应有可能是因为高频率的请求导致的）
    if response.status != 200 and new_delay <= slot.delay:
        return


    slot.delay = new_delay