Python-爬取各种开源软件的官方仓库历史版本源代码及相关信息

爬取各种开源软件的官方仓库历史版本源代码及相关信息

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

from scrapy import Item Field


class OpensourceSpiderItem（Item）:
    # define the fields for your item here like:
    # name = scrapy.Field（）
    downurl  = Field（）
    filesize = Field（）
    orginname= Field（）

 属性            大小     日期    时间   名称
----------- ---------  ---------- -----  ----
     目录           0  2016-11-04 05:56  opensource_spider-master\
     文件         423  2016-11-04 05:56  opensource_spider-master\.gitignore
     文件         640  2016-11-04 05:56  opensource_spider-master\README.md
     目录           0  2016-11-04 05:56  opensource_spider-master\opensource_spider\
     文件           0  2016-11-04 05:56  opensource_spider-master\opensource_spider\__init__.py
     文件         365  2016-11-04 05:56  opensource_spider-master\opensource_spider\items.py
     文件        3738  2016-11-04 05:56  opensource_spider-master\opensource_spider\pipelines.py
     文件        3851  2016-11-04 05:56  opensource_spider-master\opensource_spider\settings.py
     目录           0  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\
     文件         161  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\__init__.py
     文件        2464  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_apache_spider.py
     文件        2188  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_curl_spider.py
     文件        2171  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_ffmpeg_spider.py
     文件        2206  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_git_spider.py
     文件        2173  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_gnu_spider.py
     文件        2188  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_isc_spider.py
     文件        2207  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_kernel_spider.py
     文件        2291  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_lvs_spider.py
     文件        1640  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_nginx_spider.py
     文件        2249  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_python_spider.py
     文件        1764  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_sphinx_spider.py
     文件        1675  2016-11-04 05:56  opensource_spider-master\opensource_spider\spiders\opensource_vim_spider.py
     文件         278  2016-11-04 05:56  opensource_spider-master\scrapy.cfg