Indeed Web Crawler by Python Version 2

import pip

def install(package):
   pip.main(['install', package])

# install Natural language toolkit
install('nltk')

nltk.download('punkt')
nltk.download('stopwords')
nltk.download('averaged_perceptron_tagger')

# import package

import requests
import bs4
from bs4 import BeautifulSoup
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import nltk

# Demo Code and Function

# url
url ='https://ca.indeed.com/jobs?q=data&l=Ontario'

# get the url
page = requests.get(url)

#decode with BeautifulSoup
soup = BeautifulSoup(page.text, 'html.parser')

# job title is under div/a node with data-tn-element/title label
def job_title(soup):
  jobs = []
  for div in soup.find_all(name='div', attrs={'class':'row'}):
    for a in div.find_all(name='a', attrs={'data-tn-element':'jobTitle'}):
        jobs.append(a['title'])
  return(jobs)

# This company code is from : https://medium.com/@msalmon00/web-scraping-job-postings-from-indeed-96bd588dcb4b

# company title : <span class="company"> in div node
def company(soup):
    companies = []
    for div in soup.find_all(name='div', attrs={'class':'row'}):
        company = div.find_all(name='span', attrs={'class':'company'})
        # if can be select by company
        if len(company) > 0:
            for b in company:
                companies.append(b.text.strip())
        # else pick up by result-link-source
        else:
            sec_try = div.find_all(name='span', attrs={'class':'result-link-source'})
            for span in sec_try:
                companies.append(span.text.strip())
    return(companies)

# location
def location(soup):
  locations = []
  spans = soup.find_all('span', attrs={'class':'location'})
  for span in spans:
      locations.append(span.text)
  return(locations)

# Salary : most of salary doens't exist, fill NA
def salary(soup):
  salaries = []
  for div in soup.find_all(name='div', attrs={'class':'row'}):
    try:
      salaries.append(div.find(name = 'span',attrs = {'class','no-wrap'}).text.strip())
    except:
      salaries.append('Nothing_found')
  return(salaries)

# Post Day
def post_day(soup):
  postday = []
  for div in soup.find_all(name = 'div',attrs = {'class','result-link-bar'}):
      try :
          postday.append(div.find('span',attrs={'class': 'date'}).text)
  #spans = soup.findAll('span', attrs={'class': 'date'})
      except :
          postday.append('New Post')
  return(postday)

# reviews
def review(soup):
  review = []
  for div in soup.find_all(name='div', attrs={'class':'row'}):
    try:
      review.append(div.find(name = 'span',attrs = {'class','slNoUnderline'}).text.strip())
    except:
      review.append('0 reviews')
  return(review)

# post functions : what we get is the absolute address, so we need to add 'ca.indeed.com'
def post_link(soup):
  link = []
  for div in soup.find_all(name='div', attrs={'class':'row'}):
    for a in div.find_all(name='a', attrs={'data-tn-element':'jobTitle'}):
        link.append('http://ca.indeed.com'+a['href'])
  return(link)

# concat all
columns = {'job_title' : job_title(soup), 'company_name' : company(soup),  
           'location' : location(soup), 'salary' : salary(soup),
           'review': review(soup), 'post day' : post_day(soup),
           'links': post_link(soup)
           }
sample_df = pd.DataFrame(columns)
print(sample_df)

# pickup usful information from linked pages
# info
words = []
def info(link) :
    soup = BeautifulSoup(requests.get(link).text,'html.parser')
    words.extend(soup.find('span',attrs={'id': 'job_summary'}).text.split())
    return(words)

# demo
info('https://ca.indeed.com/viewjob?jk=53d052c5a4791a68&tk=1c77cg3ku41dk895&from=serp&alid=3&advn=6591858045400699')[:10]

# There are lots of meanless words, try to drop them by stop words list

# url
url ='https://ca.indeed.com/jobs?q=data&l=Ontario&start='

# loop pages
page = [i*10 for i in range(0,51)]

# new dataframe for data
df = pd.DataFrame()#columns = columns)

# loop 50 pages
for i in page :
    url =  url + str(i)  

    # get the url
    page = requests.get(url)

    #specifying a desired format of “page” using the html parser - this allows python to read the various components of the page, rather than treating it as one long string.
    soup = BeautifulSoup(page.text, 'html.parser')

    # concat all
    temp = pd.DataFrame([job_title(soup),company(soup),location(soup),
                         salary(soup),review(soup),post_day(soup),post_link(soup)])
    df = df.append(temp.T,ignore_index = True)

df.columns = ['job_title', 'company_name', 'location', 'salary', 'review',
          'post day','links']
print(df.head())
print('-'*80)
print('Total Data we have now: ',len(df))

# Job title
# job title : split first title outside by '-' , ',','?','$'
df['First_title'] = df['job_title'].map(lambda x : x.split('–')[0].split(',')[0].split('(')[0].split('$')[0].split('-')[0].strip())

# top 15 job titles
df['First_title'].value_counts()[:15].plot.barh(title = 'Top 15 Most Wanted Job Titles \n Ordered Ascendingly',
                                                colormap = 'summer')
plt.show()

# salary

# most of salary are 'Nothing_found', replace by 0
df['salary'] = df['salary'].replace('Nothing_found','0')

# make all salary / year
# since most salary exists is a range, get mean value of salary
mean_salary = []
for s in df['salary'] :
    if s != '0' :
        min_sal = float(s.split()[0].replace('$','').replace(',',''))
        if s.split()[1] != '-' :
            mean_sal = min_sal
            #print(mean_sal)
            unit = s.split()[-1]
            # print(mean_sal,unit)
            if unit == 'week' :
                # assume 52 weeks / year
                mean_salary.append(mean_sal * 52)
            elif unit == 'month' :
                mean_salary.append(mean_sal * 12)
            elif unit == 'hour':
                # assume 40 hours/ week
                mean_salary.append(mean_sal * 40*52)
            else :
                mean_salary.append(mean_sal)    
        else :
            max_sal = float(s.split()[2].replace('$','').replace(',',''))
            mean_sal = (min_sal+max_sal)/2
            #print(mean_sal)
            unit = s.split()[-1]
            # print(mean_sal,type(mean_sal),unit)
            if unit == 'week' :
                # assume 52 weeks / year
                mean_salary.append(mean_sal * 52)
                #print(df['mean_sal'],'in df')
            elif unit == 'month' :
                mean_salary.append(mean_sal * 12)
            elif unit == 'hour':
                # assume 40 hours/ week
                mean_salary.append(mean_sal * 40*52)
            else :
                mean_salary.append(mean_sal)
    else :
        mean_salary.append(0)

len(mean_salary)       

df['mean_sal'] = mean_salary


# number of reviews

df['review number'] = df['review'].map(lambda x : int(x.split()[0].replace(',','')))

#Visualization

# salary visualization
g = sns.kdeplot(df['mean_sal'],shade = True,color = 'g')
g.set_title('Mean Salary for data-related jobs')
plt.show()

df['mean_sal'].value_counts()

# company name

# graph the top 20 companies who needed the data-related employees recently
df['company_name'].value_counts()[:20].plot.barh(title = 'Top 20 Companies Who Need Data-related Employees by Feb/25/2018',
                                                colormap = 'summer')
plt.show()

# reviews visualization
# plot top 10 companies with most reviews
df[['review number','company_name']].groupby('company_name').mean().sort_values(by = ['review number'],
  ascending = False).head(10).plot.barh(title = 'Top 10 companies with most reviews \n Who Need Employees Recently',
                                        colormap = 'summer')
plt.show()

# Do more work on link column, check more details key features
import time
for l in df['links'] :

    # get all words for linked pages by function info
    info(l)

    # stop 0.5s for each loop
    time.sleep(0.5)

print('Total words now: ',len(words))
print('-'*40)
print('Unique words now: ',len(set(words)))

# stop words
stopwords = nltk.corpus.stopwords.words('english')

# drop words exist in stopwords
keywords = []
for word in words :
    if word.lower() not in stopwords :

        # drop the words whose length is less than 3
        if len(word) >= 3 :
            keywords.append(word.lower())

print('Total keywords now: ',len(keywords))   
print('-'*40)
print('Unique Keywords now: ',len(set(keywords)))

# Pos tag words
word_tag = nltk.pos_tag(keywords)

# pick up tags of NN
nouns = [i for i,j in nltk.pos_tag(keywords) if j == 'NN']

print('Total Nouns now: ',len(nouns))   
print('-'*40)
print('Unique Nouns now: ',len(set(nouns)))

pd.Series(nouns).value_counts().head(15).plot.barh(title = 'Top 15 Words in job description',colormap = 'summer')
plt.show()

# pick up tags of VB
verbs = [i for i,j in nltk.pos_tag(keywords) if j in ['VB','VBD']]

print('Total Nouns now: ',len(verbs))   
print('-'*40)
print('Unique Nouns now: ',len(set(verbs)))

pd.Series(verbs).value_counts().head(15).plot.barh(title = 'Top 15 Words in job action',colormap = 'summer')
plt.show()

# list to csv and make visualization via tableau
pd.DataFrame(nouns,columns = ['nouns']).to_csv('keyword.csv',index = False)

# list to csv and make visualization via tableau
pd.DataFrame(verbs,columns = ['verbs']).to_csv('keyword.csv',index = False)

# list to csv and make visualization via tableau
pd.DataFrame(df).to_csv('indeed.csv',index = False)