Парсинг на scrapy

from scrapy.spider import Spider
from scrapy.selector import Selector
from scrapy.http import Request
from workopolis.items import Node
 
import time
import re
 
 
 
 
 
 
 
 
class apply(Spider):
 
    name = "apply_spider"
 
    lang = ''
 
    rowFrom = ''
 
    location = ''
 
    title = ''
 
    city = ''
 
    description = ''
 
    category = ''
 
    job_id = ''
 
    driver = None
 
 
 
    start_urls = [
        'https://apply.firstgroupcareers.com/vacancy/find/results/'
        ]
    
    url_app = 'https://apply.firstgroupcareers.com/vacancy/jobecode/description/'
 
 
 
    def parse(self, response):
        
        hxs = Selector(response)
        
        jobs = hxs.xpath('//*[@class="rowContainerHolder"]')
        items = []
        
        for job in jobs:
            item = Node()
            x = job
 
        
 
        item['title'] = hxs.xpath("string(.//*[contains(@class,'rowHeader')])").extract()[0]
 
        url = html.xpath("string(.//*[contains(@class,'rcMenu')]//a").extract()[0]
 
 
        item['job_id'] = re.search('jobId=([0-9]+)', url).groups()[0]
 
        item['apply_url'] = self.url_app.replace('jobecode',item['job_id'])
 
        g = item['title'].split('-')
 
        item['title'] = g[1]
 
        item['city'] = g[0]
 
                        
                    
        print   item['apply_url']
                    
        d_url = item['apply_url']
 
 
                
 
                
                    
 
        request = Request(d_url, callback=self.parse_details)
        request.meta['item'] = item
        items.append(request)
                
 
        for item in items:
            
            yield item
 
 
 
 
    def parse_details(self,response):
 
        html = Selector(response)
        item = response.meta['item']
 
        item['description'] = html.xpath(".//*[contains(@class,'earcu_posdescriptionnote')]").extract()[0]
 
 
        item['language'] = 'en'
 
        item['state'] = html.xpath(".//*[contains(@class,'jobSumValue')]").extract()[0]
        
        
        
 
        
        return item