146 lines
3.9 KiB
Python
146 lines
3.9 KiB
Python
from gettags import get_tags
|
|
from progress.spinner import PixelSpinner
|
|
from bs4 import BeautifulSoup
|
|
import yaml
|
|
import requests
|
|
import fake_useragent
|
|
import time
|
|
import threading
|
|
import random
|
|
import queue
|
|
import os
|
|
import json
|
|
import csv
|
|
import gevent.monkey
|
|
gevent.monkey.patch_all()
|
|
|
|
|
|
with open('set.yaml', 'r') as f:
|
|
cookie = yaml.load(f, Loader=yaml.CLoader)["cookid"]
|
|
# setting
|
|
url = "https://nhentai.net/favorites/"
|
|
apiurl = "https://nhentai.net/api/gallery/"
|
|
table = [
|
|
["id", "name", "tags"]
|
|
]
|
|
now = 1
|
|
allnumbers = []
|
|
allnames = []
|
|
alltags = []
|
|
ua = fake_useragent.UserAgent()
|
|
useragent = ua.random
|
|
|
|
# request
|
|
def wtfcloudflare(url,method="get",data=None):
|
|
session = requests.Session()
|
|
session.headers = {
|
|
'Referer': "https://nhentai.net/login/",
|
|
'User-Agent': "",
|
|
'Cookie': cookie,
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
}
|
|
if method == "get":
|
|
r = session.get(url)
|
|
elif method == "post":
|
|
r = session.post(url,data=data)
|
|
return r
|
|
|
|
|
|
|
|
class gettagonline(threading.Thread):
|
|
def __init__(self, queue, number):
|
|
threading.Thread.__init__(self)
|
|
self.number = number
|
|
self.queue = queue
|
|
|
|
# def run(self):
|
|
# while self.queue.qsize() > 0:
|
|
# num = self.queue.get()
|
|
# # print("get %d: %s" % (self.number, num))
|
|
# ua = fake_useragent.UserAgent()
|
|
# useragent = ua.random
|
|
# headers = {
|
|
# 'user-agent': useragent
|
|
# }
|
|
# r = requests.get(apiurl + num, headers=headers)
|
|
# data = r.json()
|
|
# ctag = []
|
|
# for i in enumerate(data['tags']):
|
|
# ctag.append(i[1]['name'])
|
|
# alltags.append(ctag)
|
|
# time.sleep(random.uniform(0.5, 1))
|
|
|
|
|
|
set1 = input("請問要使用離線資料嗎?(y/n)(默認為否)")
|
|
if set1 == "y".lower() or set1 == "yes".lower():
|
|
if not os.path.isfile("tag.json"):
|
|
print("沒有發現離線資料 抓取中請稍後...")
|
|
get_tags()
|
|
print("抓取完畢")
|
|
print("使用離線資料")
|
|
else:
|
|
print("使用線上資料")
|
|
threadscount = input("請輸入要使用幾個線程(默認為5 不可超過10)")
|
|
if threadscount == "":
|
|
threadscount = 5
|
|
else:
|
|
try:
|
|
threadscount = int(threadscount)
|
|
if threadscount > 10:
|
|
threadscount = 10
|
|
except:
|
|
threadscount = 5
|
|
|
|
spinner = PixelSpinner('抓取資料中...')
|
|
while True:
|
|
data = wtfcloudflare(f"{url}?page={now}")
|
|
soup = BeautifulSoup(data.text, 'html.parser')
|
|
book = soup.find_all("div", class_='gallery-favorite')
|
|
if book == []:
|
|
break
|
|
numbers = [t.get('data-id') for t in book]
|
|
names = [t.find('div', class_="caption").get_text() for t in book]
|
|
tags_ = [t.find('div', class_="gallery").get('data-tags') for t in book]
|
|
tags = []
|
|
for i in tags_:
|
|
tags__ = i.split(' ')
|
|
tags.append(tags__)
|
|
allnumbers.extend(numbers)
|
|
allnames.extend(names)
|
|
alltags.extend(tags)
|
|
now += 1
|
|
spinner.next()
|
|
|
|
|
|
if set1 == "y".lower() or set1 == "yes".lower():
|
|
with open('tag.json', 'r') as f:
|
|
tagjson = json.load(f)
|
|
for i in enumerate(allnumbers):
|
|
tagstr = ""
|
|
for j in alltags[i[0]]:
|
|
if j in tagjson:
|
|
tagstr += tagjson[j] + ", "
|
|
|
|
table.append([i[1], allnames[i[0]], tagstr])
|
|
else:
|
|
alltags = [] # 清空
|
|
get_tags_queue = queue.Queue()
|
|
threads = []
|
|
for i in allnumbers:
|
|
get_tags_queue.put(i)
|
|
for i in range(threadscount):
|
|
t = gettagonline(get_tags_queue, i)
|
|
t.start()
|
|
threads.append(t)
|
|
for t in threads:
|
|
t.join()
|
|
|
|
for i in enumerate(allnumbers):
|
|
table.append([i[1], allnames[i[0]], alltags[i[0]]])
|
|
|
|
|
|
with open('output.csv', 'w', newline='', encoding="utf_8_sig") as csvfile:
|
|
writer = csv.writer(csvfile)
|
|
writer.writerows(table)
|