[爬虫]python根据职位或者公司关键字爬取某招聘网站招聘数据并存储到数据库和exeal中

import requests# 模拟乞求 post get from Vpinyin import Pinyin # 将中文输出为拼音 import pprint # 格局化输出模块 import os # 收配文件和目录 import re # 正则表达式模块 内置 import json import csZZZ # 输出csZZZ的库 import pymysql # 数据库 import time # 计较机内部光阳库 session = requests.session()# 为了保持代码中所有的session统一 def url_get(data,i):#界说URL拼接 url_1 = 'hts://search.51jobss/list/000000,000000,0000,00,9,99,' url_2 = ',2,' url_yema = str(i) url_3 = '.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&' \ 'jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=' url = url_1 +data+url_2+url_yema+url_3 return (url) # 代码运止 data = str(input("请选择爬与职位大概公司： ")) Vinzi_1 = []# 薪资列表 数据1 Vinzi_2 = []# 薪资列表 数据2 Vinzi_3 = []# 薪资列表 元的单位 返回10000 or 1000 Vinzi_4 = []# 薪资列表 年or月单位 citf_l = []# 都市列表 eVp_l = [] # 经历要求 edu_l = [] # 学历要求 count_people = 0 # 统计总爬与有效数据个数 count_Vinzi = 0 # 统计总爬与薪资有效数据 Vinzi_count1 = float(0) # 将如0.6-1万/月薪资的0.6转为float型，便于计较 Vinzi_count2 = float(0) # 将如0.6-1万/月薪资的1转为float型，便于计较 citf_d = {} # 字典 eVp_d = {} # 经历要求 edu_d = {} # 学历要求 work_gs = [] work_href = [] # 工做具体网址列表，便于二次循环爬与 f_file = data+"/"+"雇用_"+data + ".csZZZ" # 界说依据输入的data自界说文件名并翻开的变质 # 创立data文件夹 path = "D:/进修/东软/大三上期/脚原语言开发（陈汉斌）/yunVingDaiMa/雇用/"+data if not os.path.eVists(path): os.mkdir(path) f = open(f_file, mode='w+', encoding='ANSI', newline='') # ANSI为css的eVeal编码 utf-8为tVt编码 hts://ss.runoobss/python/python-func-open.html csZZZ_writer = csZZZ.DictWriter(f, ['公司称呼', '职位称呼', '都市地区', '经历要求', '学历要求', '雇用人数','职位福利', '职位薪资', '职位发布日期', '职位详情网页'], dialect='eVcel') csZZZ_writer.writeheader() # 写入表头 # 判断输入语句能否为中文，是中文变质a为'job_zhiwei_'+ result，不是a为'job_zhiwei_' + data，a是数据库名 # 参考网址： hts://blog.csdn.net/Kobe123brant/article/details/110326353?ops_request_misc=%257B%2522request%255Fid%2522%253A%2522163990869516780357286336%2522%252C%2522scm%2522%253A%252220140713.130102334.pc%255Fall.%2522%257D&request_id=163990869516780357286336&biz_id=0&utm_medium=distribute.pc_search_result.none-task-blog-2~all~first_rank_ecpm_ZZZ1~rank_ZZZ31_ecpm-2-110326353.pc_search_result_cache&utm_term=python%E5%B0%86%E4%B8%AD%E6%96%87%E8%BD%AC%E6%8D%A2%E4%B8%BA%E6%8B%BC%E9%9F%B3%E7%9A%84%E6%96%B9%E6%B3%95&spm=1018.2226.3001.4187 # 数据库名预界说函数，不须要数据库可增除以下代码 def is_Chinese(word): for ch in word: if '\u4e00' <= ch <= '\u9fff': return True return False if is_Chinese(data) : s = Pinyin().get_pinyin(data).split('-') result = ''.join([i[0]. upper() for i in s]) # 中文转换为首字母 如大数据：DSJ a = 'job_zhiwei_'+ result# 真现中文转义 else: a = 'job_zhiwei_' + data # print("******************************测试收解以下为数据库变质名字a************************************") # print(a) # 连贯数据库 conn = pymysql.connect(host="localhost", port=3306,user="root",password="",database="new_wen",charset='utf8') cursor = conn.cursor()# 界说数据库游标 #创立表job_zhiwei_'data'[ 'id', '公司称呼', '职位称呼', '都市地区', '经历要求', '学历要求', '雇用人数','职位福利', '职位薪资', '职位发布日期', '职位详情网页'] sql_create = """create table %s ( id int(4) NOT NULL AUTO_INCREMENT , ompany_name ZZZarchar(100) not null, job_name ZZZarchar(100) not null, job_area ZZZarchar(50) not null, job_eVp ZZZarchar(20) not null, job_edu ZZZarchar(20) not null, recruiting_numbers ZZZarchar(20) not null, job_weal ZZZarchar(200) null, job_pay ZZZarchar(50) null, job_release_date ZZZarchar(20) not null, job_webpage ZZZarchar(500) not null, PRIMARY KEY (id) )""" %(a) cursor.eVecute(sql_create) # 执止创立数据库 cookie_i1 = -1 # 此处cookie须要手动添加5个 cookie_j1 = [ '_uab_collina=164009531079133854257018; guid=b61b9790083897515ead52b430fa7f7c; nsearch=jobarea%3D%26%7C%26ord_field%3D%26%7C%26recentSearch0%3D%26%7C%26recentSearch1%3D%26%7C%26recentSearch2%3D%26%7C%26recentSearch3%3D%26%7C%26recentSearch4%3D%26%7C%26collapse_eVpansion%3D; search=jobarea%7E%60000000%7C%21ord_field%7E%600%7C%21recentSearch0%7E%60000000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21recentSearch1%7E%60000000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%CA%FD%BE%DD%BF%AA%B7%A2%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21recentSearch2%7E%60000000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FAcss%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21recentSearch3%7E%60000000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%B4%F3%CA%FD%BE%DD%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21; acw_tc=76b20fed16401872784651156e73596d44930468e6d412e280ea9ea5b4f281; acw_sc__ZZZ2=61c3458e4cf4f427e44da3d4d2b1abe8417b4db4; ssVmod_itna=eqUV9DgD0DuDn7DRGDzgKnh2EKDCY+lWWCCRSRDBkDAK3DZDiqAPGhDC++9Om3brmF3DE8YwQ677ipZZZ=j8rApfqYWh0h3txADB3DEV0=KrRWKiiBDCeDIDWeDiDG4Gm4qGtDpVG=gDm4GWGqGfDDoDY4tcDitD4qDBmgdDKqGgGuDKWDUqx82lIltKteVyDDNBD0U3VBdK71u5kZaPDWPcDB=CVBjZRqtMIeDH/oLYtrhslG+Et/h5YDVsmSVo77GT+0DNCGGqe3q3bzCDGb82EGVVD=; ssVmod_itna2=eqUV9DgD0DuDn7DRGDzgKnh2EKDCY+lWWCCRSD8dpgVGX=G8DFgwOUc2aOIyoRPGF+nkKVK45WCDeNMI=eZZZUG3qS7KZZZmi=G=I9GeIPuGi+VE20oaAhxllC=4X5CLckNu+iI5azQWlcaU25HrBPseE2kc65WyQGHyePTU7Ib3bu8UogpmP9mR2pImhZZZIYB44tFwtC2w+Q6nEegfHqQk=mUmCnBfChgfTHcLdl+d2dr9=nS+02agbtpA2bMInrAuptLmLY6ZfUCT+zrnwR1q=3zwGIdI2HcIjgaTm7awT36/upxQSmrT7rDoeEsUioo4iehdiS7qYa20oURYNhYRh5o4YlD1ie5KuN1Wz9+QL2N2DowWQ5fYNloFQKzE7Kfbl0+ZZZ3=joNwL34bFIricf3N4Yr0QATAD1zRe3AkClOsPD3gTqffDPWbo+Q98ftoawRPs3rRg38SpigbVLjcYzUb5+aoXGNHbtFb3PjYtiUidPD7QDVGcDG7DfVz0DywDYiDD===', '_uab_collina=164009531079133854257018; guid=b61b9790083897515ead52b430fa7f7c; nsearch=jobarea%3D%26%7C%26ord_field%3D%26%7C%26recentSearch0%3D%26%7C%26recentSearch1%3D%26%7C%26recentSearch2%3D%26%7C%26recentSearch3%3D%26%7C%26recentSearch4%3D%26%7C%26collapse_eVpansion%3D; search=jobarea%7E%60000000%7C%21ord_field%7E%600%7C%21recentSearch0%7E%60000000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21recentSearch1%7E%60000000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%CA%FD%BE%DD%BF%AA%B7%A2%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21recentSearch2%7E%60000000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FAcss%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21recentSearch3%7E%60000000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%B4%F3%CA%FD%BE%DD%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21; acw_tc=76b20fed16401872784651156e73596d44930468e6d412e280ea9ea5b4f281; acw_sc__ZZZ2=61c3458e4cf4f427e44da3d4d2b1abe8417b4db4; ssVmod_itna=QuG=BKGKD5emVDq0LeYKO6mGcc2DVmqmwn4KDsaDcYVA5D8D6DQeGTbRQTrtLiziZkDIaKicYhfa5FhSihiA4ECwmbDU4i8DCkiE3pDem=D5VGoDPVDeDADYoUDAqiOD7L=DEDmR8DaVDbDin8pVGCDeKD0VwFDQKDu6FVqD284fphngACqY7YEVDzLVG1i40HqCfp3+ffij53pV0ku40O9ry82gYDUCpbtmGelC+T4Ni4osiYdQrToHGqqW20tW04oCDW/GV2sqVDicReqYD===; ssVmod_itna2=QuG=BKGKD5emVDq0LeYKO6mGcc2DVmqmwn4ikZZZqhqDlEaDjbpublfk6ZG==i=P8VApq2eFoI+FY5Qkr+hYrSYRoDwVEPGcDYFFtD1aOlVVD=', '', '', '' ] for i in range(1, 21): # range(1, 31)为循环1-30页 url = url_get(data, i) # 挪用url_get函数拼接url # 一个cookie用8页 if i % 8 == 1: cookie_i1 += 1 print("cookie={}".format(cookie_i1)) if cookie_i1 == 2: break else: headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.20 Safari/537.36', 'Accept-Encoding': 'gzip, deflate, br', 'Cookie':cookie_j1[cookie_i1] } time.sleep(5) response = session.get(url=url, headers=headers) # 模拟get乞求 返复书息response对象 print("********************正正在爬与第{}页********************".format(i)) # 动态网页 response.json()获与json字典数据 # 保存下载图片 室屏 音频 获与响应体二进制数据 response.content() #print(response.teVt) # 获与数据 获与响应文原的文原数据 reponse.test html_date = re.findall('window.__SEARCH_RESULT__ = (.*?)</script>',response.teVt)[0] #print(html_date) # 把字符串数据转换为json字典数据 json_date = json.loads(html_date) #格局化输出（都雅） # pprint.pprint(json_date) #解析json数据，写成键值对方式 search_esult = json_date['engine_jds'] #for遍历 提与列表中元素 for indeV in search_esult: #pprint.pprint(indeV) company_name = indeV["company_name"] # 公司称呼 title = indeV["job_name"] # 职位称呼 info_list = indeV["attribute_teVt"] #职位要求根柢信息如['成都', '3-4年经历', '原科', '招1人'] if len(info_list) == 4: # 将薪资根柢信息彻底的才输出 citf = info_list[0] # 都市 eVp = info_list[1] # 经历要求 edu = info_list[2] # 学历要求 people = info_list[3] # 雇用人数 I = info_list[0][0:2] # 大地区获与 citf_l = citf_l + [I] eVp_l = eVp_l + [eVp] edu_l = edu_l + [edu] #列表转为字符串 空格收解 #str(attribute_teVt) count_people= count_people+1 # 统计有效爬与人数 #job_info = '|'.join(info_list) # 可以将列表转换为字符串，以|标记收解 jobwelf = indeV["jobwelf"] # 职位福利 money = indeV["proZZZidesalary_teVt"] # 职位薪资 4-4.5千/月 1.5-2.8万/月 15-20万/年 updatedate = indeV["updatedate"] # 职位发布日期 job_href = indeV["job_href"] # 职位详情网页 work_href = work_href + [job_href] # 创立信息字典 键只能是元组、数字、字符串 dit = { '公司称呼': company_name, '职位称呼': title, '都市地区': citf, '经历要求': eVp, '学历要求': edu, '雇用人数': people, '职位福利': jobwelf, '职位薪资': money, '职位发布日期': updatedate, '职位详情网页': job_href, } csZZZ_writer.writerow(dit) sql_insert = "insert into %s (ompany_name,job_name, job_area,job_eVp,job_edu, " \ "recruiting_numbers,job_weal,job_pay,job_release_date,job_webpage) " \ "ZZZalues('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')" % (a, company_name, title, citf, eVp, edu, people, jobwelf, money, updatedate, job_href) cursor.eVecute(sql_insert) connssmit() # 提交乞求，不然不会插入数据 print(dit) ### 薪资阐明代码 if money == '' or money[money.find("/") - 2:-3:1] == "以" or [money[-1:]] == ["天"] or [money[-1:]] == ["时"]: # 牌除烦扰项 1.5千以下/月、100万以上/年 pass else: count_Vinzi += 1 a1 = money.find("-") a2 = money.find("/") work_gs+=[company_name] Vinzi_1 = Vinzi_1 + [money[0:a1]] # 钱数字1 Vinzi_2 = Vinzi_2 + [money[a1 + 1:-3]] # 钱数字2 if [money[a2 - 1:-1]] == ["万/"]: # 输出单位万/ or 千/的数列代码 d1 = float(10000) else: d1 = float(1000) if [money[-1:]] == ["月"]: # 输出单位年的数列代码 d2 = float(1) elif [money[-1:]] == ["年"]: d2 = float(12) Vinzi_3 = Vinzi_3 + [d1] # 输出单位万/&千/ 的值10000 or 1000 Vinzi_4 = Vinzi_4 + [d2] # 输出单位月和年的值1大概12 #break # 调试单次循环，便于查察 # 雇用数据、阐明写入 字典数据牌序(内容，要害点=匿名函数以V中第二个元素为范例牌序，给取逆序)[地区列表] cursor.close() # 封锁数据库链接 conn.close() def dict_analyse(l,d,k,ZZZ,z):# 创立字典阐明办法 f_dqfb_file = data+"/"+data + "_" + z + "阐明表.csZZZ" f = open(f_dqfb_file, mode='w+', encoding='ANSI', newline='') # ANSI为eVeal编码 utf-8为tVt编码 hts://ss.runoobss/python/python-func-open.html csZZZ_writer = csZZZ.DictWriter(f, [k, ZZZ], dialect='eVcel') csZZZ_writer.writeheader() # 写入表头 for i in l:#阐明区域分布，结果为字典 d[i] = d.get(i,0)+1 #print(d) for i,j in sorted(d.items(), key=lambda V:V[1], reZZZerse=True):# ZZZalue值降序牌序并写入csZZZ中 dit2 = { k: i, ZZZ: j } csZZZ_writer.writerow(dit2) dict_analyse(citf_l, citf_d, k="雇用地区", ZZZ="雇用数质",z="地区") # 挪用执止dict_analyse函数，写入scZZZ中 dict_analyse(eVp_l, eVp_d, k="经历要求", ZZZ="数质",z="经历") dict_analyse(edu_l, edu_d, k="学历要求", ZZZ="数质",z="学历") # 均匀薪资阐明 Vinzi_5 = [] # 此处预界说一个空列表，将薪资阐明的数据写入到csZZZ文件中，以便进一步阐明 Vinzi_6 = [] # num_list1 = [float(i) for i in Vinzi_1]#将薪资数据的队列string型转为float型 num_list2 = [float(i) for i in Vinzi_2] for i in range(count_Vinzi): j1 = num_list1[i]*Vinzi_3[i]/Vinzi_4[i] j2 = num_list2[i]*Vinzi_3[i]/Vinzi_4[i] Vinzi_5+=[round(j1/1000, 1)] # 将薪资转换为K单位的薪资，如6.2K，输出6.2，并糊口生涯1位小数 Vinzi_6+=[round(j2/1000, 1)] Vinzi_count1 = Vinzi_count1+j1 Vinzi_count2 = Vinzi_count2+j2 aZZZerage_wage1 = Vinzi_count1/count_Vinzi aZZZerage_wage2 = Vinzi_count2/count_Vinzi f_fVinzi = data + "/" + data + "_" + "薪资" + "阐明表.csZZZ" f = open(f_fVinzi, mode='w+', encoding='ANSI',newline='') csZZZ_writer = csZZZ.DictWriter(f, ['公司', '薪资数据1','薪资数据2','中位薪资'], dialect='eVcel') csZZZ_writer.writeheader() # 写入表头 for i in range(count_Vinzi): Vinzi_zw = round((Vinzi_5[i]+Vinzi_6[i])/2, 1) dit3 = { '公司': work_gs[i], '薪资数据1': Vinzi_5[i], '薪资数据2': Vinzi_6[i], '中位薪资': Vinzi_zw, } csZZZ_writer.writerow(dit3) # 输出均匀薪资的计较结果 print("原次爬与有效职位共{}个，爬与有效薪资的数据为{}个，该职位或该公司的均匀薪资为{}-{}元/月". format(count_people, count_Vinzi, round(aZZZerage_wage1, 0), round(aZZZerage_wage2, 0)))