Scrapy递归下载内容

敲了好几次头之后，我终于来到这里了。

问题：我正在尝试下载每个 craiglist 帖子的内容。我所说的内容是指“发布正文”，例如手机的描述。自 iPhone 以来寻找新的旧手机已经令人兴奋不已。

该代码是一项很棒的工作迈克尔·赫尔曼.

我的蜘蛛班

from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import *
from craig.items import CraiglistSampleItem

class MySpider(CrawlSpider):
    name = "craigs"
    allowed_domains = ["craigslist.org"]
    start_urls = ["http://minneapolis.craigslist.org/moa/"]

    rules = (Rule (SgmlLinkExtractor(allow=("index\d00\.html", ),restrict_xpaths=('//p[@class="nextpage"]',))
    , callback="parse_items", follow= True),
    )

    def parse_items(self,response):
        hxs = HtmlXPathSelector(response)
        titles = hxs.select("//span[@class='pl']")
        items = []
        for titles in titles:
            item = CraiglistSampleItem()
            item ["title"] = titles.select("a/text()").extract()
            item ["link"] = titles.select("a/@href").extract()
            items.append(item)
        return items

和 Item 类

from scrapy.item import Item, Field

class CraiglistSampleItem(Item):
    title = Field()
    link = Field()

由于代码将遍历许多链接，因此我想将每部手机的描述保存在单独的 csv 中，但 csv 中多一列也可以。

任何线索！

而不是退回物品parse_items你应该返回/产生 scrapy 的方法Request实例以便从项目页面获取描述，link and title你可以通过一个Item, and Item里面的meta字典：

from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.http import Request
from scrapy.selector import *

from scrapy.item import Item, Field


class CraiglistSampleItem(Item):
    title = Field()
    link = Field()
    description = Field()


class MySpider(CrawlSpider):
    name = "craigs"
    allowed_domains = ["craigslist.org"]
    start_urls = ["http://minneapolis.craigslist.org/moa/"]

    rules = (Rule(SgmlLinkExtractor(allow=("index\d00\.html", ), restrict_xpaths=('//p[@class="nextpage"]',))
        , callback="parse_items", follow=True),
    )

    def parse_items(self, response):
        hxs = HtmlXPathSelector(response)

        titles = hxs.select("//span[@class='pl']")
        for title in titles:
            item = CraiglistSampleItem()
            item["title"] = title.select("a/text()").extract()[0]
            item["link"] = title.select("a/@href").extract()[0]

            url = "http://minneapolis.craigslist.org%s" % item["link"]
            yield Request(url=url, meta={'item': item}, callback=self.parse_item_page)

    def parse_item_page(self, response):
        hxs = HtmlXPathSelector(response)

        item = response.meta['item']
        item['description'] = hxs.select('//section[@id="postingbody"]/text()').extract()
        return item

运行它并查看附加内容description输出 csv 文件中的列。

希望有帮助。