HQY 一个和谐有爱的空间

import requests,bs4,re,os,threading
 
 
 
class MeiNvTu:
 
    def __init__(self):
 
        self.url_main='https://网址/pw/'
 
        self.url=f'{self.url_main}thread.php?fid='
 
    def getPageMax(self,typeID=14):
 
        try:
 
            res = requests.get(f'{self.url}{typeID}')
 
            res.encoding = 'utf-8'
 
            soup = bs4.BeautifulSoup(res.text, 'lxml')
 
            pageNum = soup.select('#main > div > span.fl > div.pages.cc > span')
 
            pageNum = int(re.search('/(.*?)Go', str(pageNum)).group(1))
 
            return pageNum
 
        except:
 
            return 0
 
    def getTitleList(self,typeID=14,page=1):
 
        '''
 
        爬取栏目里某一页的列表,网络错误返回False
 
        :param typeID:
 
        :param page:
 
        :return:
 
        '''
 
        try:
 
            res=requests.get(f'{self.url}{typeID}&page={page}')
 
            res.encoding= 'utf-8'
 
            soup=bs4.BeautifulSoup(res.text,'lxml')
 
            listTitle=soup.select('tr > td > h3')
 
            lists=[]
 
            for item in listTitle:
 
                if 'html_data' in item.a['href'] :
 
                    d={}
 
                    d['href']=self.url_main+item.a['href']
 
                    d['title']=item.a.text
 
                    lists.append(d)
 
            return lists
 
        except:
 
            return False
 
    def downImg(self,url,path):
 
        '''
 
        下载一整个页面的图片
 
        :param url:
 
        :param path:
 
        :return:
 
        '''
 
        global pool_sema
 
        res = requests.get(url)
 
        res.encoding = 'utf-8'
 
        soup = bs4.BeautifulSoup(res.text, 'lxml')
 
        imgs=soup.select('#read_tpc > img')
 
        lists=[]
 
        try:
 
            for i,item in enumerate(imgs):
 
 
 
                imgUrl=re.search("window.open\('(.*?)'\);", str(item['onclick'])).group(1)
 
                imgData=requests.get(imgUrl).content
 
                typ=imgUrl.split('.')[-1]
 
                with open(f'{path}{i}.{typ}','wb')as f:
 
                    f.write(imgData)
 
        except:
 
            print('\033[31m[下载失败!网络异常] ' + path)
 
            pool_sema.release()
 
            return
 
 
 
        #将下载好的情况记录下来,下次可以跳过
 
        textpath=''
 
        for item in path.split('\\')[0:3]:
 
            textpath=textpath+item+'\\'
 
        mutex.acquire()
 
        try:
 
            with open(textpath+'log.txt','a')as f:
 
               f.writelines(path.split('\\')[3]+'\n\r')
 
        except:
 
            pass
 
        mutex.release()
 
        # 完成后线程池记录-1
 
        print('\033[31m[完成下载] '+path)
 
        pool_sema.release()
 
    def get_typeTitle(self,id):
 
        '''
 
        返回类型的标题
 
        :param id:
 
        :return:
 
        '''
 
        if id==14:
 
            return '唯美写真'
 
        if id==15:
 
            return '网友马赛克'
 
        if id==16:
 
            return '露出马赛克'
 
        if id==49:
 
            return '街拍马赛克'
 
        if id==21:
 
            return '丝袜美腿'
 
        if id==114:
 
            return '欧美马赛克'
 
    def downloadthe(self,title,path):
 
        '''
 
        判断是否已经下载过,下载过返回True,没下载过返回False
 
        :param title:
 
        :param path:
 
        :return:
 
        '''
 
 
 
        try:
 
            with open(path+'log.txt', 'r')as f:
 
                text = f.read()
 
            if title in text:
 
                return True
 
            else:
 
                return False
 
        except:
 
            return False
 
    def get_Page_History(self,path):
 
        '''
 
        读取上一次结束 的页码
 
        :param path:
 
        :return:
 
        '''
 
        try:
 
            with open(path+'pagelog.ini','r')as f:
 
                return int(f.read())
 
        except:
 
            return 0
 
if __name__ == '__main__':
 
    # 限制线程数量
 
    pool_sema = threading.BoundedSemaphore(70)
 
    # 创建互斥体
 
    mutex = threading.Lock()
 
    #创建爬取对象
 
    mnt=MeiNvTu()
 
    #栏目id
 
    typeID=21
 
    #获得最大页数
 
    page_max=mnt.getPageMax(typeID)
 
    if page_max==0:
 
        print('\033[31m网络错误!,总页数为0')
 
    else:
 
        path_main= f"D:\\爬取的网站图片\\{mnt.get_typeTitle(typeID)}\\"
 
        if os.path.isdir(path_main) != True:
 
            os.makedirs(path_main, mode=0o777)
 
        #爬取某页的列表
 
        page_History=mnt.get_Page_History(path_main)
 
        for i in range(page_max):
 
            #跳过之前下载过的页码
 
            if i+1<page_History:
 
                print(f'\033[37m跳过页码:{i + 1}')
 
                continue
 
            #记录下来页码
 
            with open(path_main+'pagelog.ini','w')as f:
 
                f.write(str(i+1))
 
            print(f'\033[37m当前页码:{i+1}')
 
            titleList = mnt.getTitleList(typeID, i + 1)
 
            if titleList==False:
 
                print('\033[31m网络错误!,列表获取失败!')
 
                break
 
            for item in titleList:
 
                title=item['title'].replace(' ','').replace(':','').replace('！','').replace('?','').replace('*','').replace('"','')
 
                path = path_main + title + "\\"
 
                #判断是否有这个目录,没有的话就建立
 
                if os.path.isdir(path) != True:
 
                    os.makedirs(path, mode=0o777)
 
                if mnt.downloadthe(title,path_main)==False:
 
                    # 线程池记录+1
 
                    pool_sema.acquire()
 
                    print('\033[37m[开始下载] '+path)
 
                    # 爬取某个标题中的所有图片
 
                    t=threading.Thread(target=mnt.downImg,args=(item['href'], path))
 
                    t.setDaemon(True)
 
                    t.start()
 
                else:
 
                    print('\033[35m发现下载过的:',title,' 已经智能跳过!')

推荐本站淘宝优惠价购买喜欢的宝贝:

本文链接：https://hqyman.cn/post/4283.html 非本站原创文章欢迎转载，原创文章需保留本站地址！

休息一下~~