[jython] JSoup Wolf.com example

1. Wolf.com example

import os
import sys
sys.path.append(r"d:\Lib\jar\jsoup-1.12.1.jar")

from java.io import *
from java.net import *
from java.util import *
from java.lang import *
from java.nio.file import *

from org.jsoup import *
from org.jsoup.nodes import *
from org.jsoup.select import *

debug = False
filter = None #"jpg"

def decodeBase64(data):
    return java.util.Base64.getDecoder().decode(data);
     
def urlToFile(urlStr, fileName, referer):
    url = URL(urlStr)
    hc  = url.openConnection()
    hc.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36")
    hc.setRequestProperty("Referer", referer);
       
    status = hc.getResponseCode();
    while status != HttpURLConnection.HTTP_OK: # and status != HttpURLConnection.HTTP_NOT_FOUND:
        if status == HttpURLConnection.HTTP_MOVED_TEMP or status == HttpURLConnection.HTTP_MOVED_PERM or status == HttpURLConnection.HTTP_SEE_OTHER:
            newUrl = hc.getHeaderField("Location")
            hc = URL(newUrl).openConnection()
            hc.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36");
            status = hc.getResponseCode();           
    Files.copy(hc.getInputStream(), Paths.get(fileName), StandardCopyOption.REPLACE_EXISTING)
    
def getJsoupDocument(url):
    while True:
        try:
            return Jsoup.connect(url).get()
        except:
            Thread.sleep(1000)

def getWolfCom(comicsUrl, baseUrl, baseDir):
    doc_toc = getJsoupDocument(comicsUrl)
    if debug: print(doc_toc.html()) #doc_toc.text()

    list = doc_toc.select("div.box > div.group.left-box > div.webtoon-bbs-list.bbs-list > ul > li")
    if debug: print(list.html())

    dir = os.path.join(baseDir,doc_toc.title().replace(":","_"))  
    if not os.path.isdir(dir): os.mkdir(dir) #dir.mkdirs()

    for e in list:
        #if( i++ < 38 ) continue;
        try:
            url = baseUrl + e.select("a").first().attr("href")
            if debug: print(url)
    
            doc_img = getJsoupDocument(url)
            imgs = doc_img.select("section.webtoon-body > div.group.image-view > img")
            print(doc_img.title())
            
            subdir = os.path.join(dir,doc_img.title().replace(":","_"))
            if not os.path.isdir(subdir): os.mkdir(subdir) #subdir.mkdirs()
    
            k = 1;
            for img in imgs:
                img_url = img.attr("src")
                if filter == None or img_url.endswith(filter):
                    if not img_url.startswith("http"):
                        img_url = baseUrl + img_url;
                    file_name = "img_%04d.jpg" % k
                    k = k + 1
                    print( img_url + " -> " + file_name )
                    urlToFile(img_url, os.path.join(subdir, file_name), comicsUrl)
        except:
            pass

if __name__ == "__main__":
    url  = "https://wfwf133.com/list?toon=1052&title=%BA%AE%BF%A1%B3%A2%C0%CE%BF%A9%C0%DA"
    iurl = "https://wfwf133.com"
    dir  = "D:/Temp2/"
    getWolfCom(url, iurl, dir)

[python] TookKor example

1. ToonKor Example

import bs4, codecs
import requests
import base64
import os
import io
 
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
image_ext = 'jpg'
request_headers = {
    'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 '
                   '(KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'),
}
 
def getFile(url):
    with codecs.open(url,'r', encoding='utf8') as f:
        html = f.read()
    return bs4.BeautifulSoup(html, 'html.parser')
 
def getUrl(url, headers={}, params=()):
    resp = requests.get(url, verify=False, headers=headers, params=params)
    #resp.headers
    #html = resp.content.decode('utf8')
    html = resp.text
    return bs4.BeautifulSoup(html, 'html.parser')
 
def getUrlHtml(url, headers={}, params=()):
    resp = requests.get(url, verify=False, headers=headers, params=params)
    return bs4.BeautifulSoup(resp.text, 'html.parser'), resp.content.decode('utf8')
 
def urlToFile(url, file_name):
    resp = requests.get(url, verify=False, headers=request_headers, params=())
    with open(file_name, "wb") as f:
        f.write(resp.content)
 
def extractTag(bs,tag):
    [s.extract() for s in bs(tag)]
 
def getToonKor( comicsUrl, baseUrl, baseDir):
    doc = getUrl(comicsUrl)
    
    table = doc.select("table.bt_view2")[0]
    elist = table.select("td.bt_title")
    title = elist[0].text
 
    table = doc.select("table.web_list")[0]
    elist = table.select("td.content__title")
 
    new_dir = os.path.join(baseDir, title.replace(":","_"))
    if not os.path.isdir(new_dir): os.mkdir(new_dir)
 
    for e in elist:
        url = baseUrl + e['data-role']
        title = e['alt']
        bs_img, html_img = getUrlHtml(url, request_headers)
        begin = html_img.index("var tnimg = '")
        end = html_img.index("';",begin)
        data = html_img[begin + 13: end]
        img_list = base64.b64decode(data.encode("UTF-8")).decode("UTF-8")
        doc_imgs = bs4.BeautifulSoup(img_list, 'html.parser')
        imgs = doc_imgs.select("img")
        
        sub_dir = os.path.join(new_dir, title.replace(":","_"))
        if not os.path.isdir(sub_dir): os.mkdir(sub_dir)
 
        k = 1;
        for img in imgs:
            img_url = img.get('src')
            if not img_url: continue
            if image_ext == None or img_url.endswith(image_ext):
                if( not img_url.startswith("http") ):
                    img_url = baseUrl + img_url
                file_name = "img_%04d.jpg" % k
                urlToFile( img_url, os.path.join( sub_dir, file_name) )
                print( img_url + " -> " + file_name )
                k = k + 1
 
if __name__ == "__main__":
        url = "https://tkr034.com/webtoon/505"
        iurl = "https://tkr034.com"
        bdir = "D:/Temp2/"
        getToonKor(url, iurl, bdir)
        print("END")

import bs4, codecs
import requests
import base64
import os
import io

requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
image_ext = 'jpg'
request_headers = {
    'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 '
                   '(KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'),
}

def getFile(url):
    with codecs.open(url,'r', encoding='utf8') as f:
        html = f.read()
    return bs4.BeautifulSoup(html, 'html.parser')

def getUrl(url, headers={}, params=()):
    resp = requests.get(url, verify=False, headers=headers, params=params)
    #resp.headers
    #html = resp.content.decode('utf8')
    html = resp.text
    return bs4.BeautifulSoup(html, 'html.parser')

def getUrlHtml(url, headers={}, params=()):
    resp = requests.get(url, verify=False, headers=headers, params=params)
    return bs4.BeautifulSoup(resp.text, 'html.parser'), resp.content.decode('utf8')

def urlToFile(url, file_name):
    resp = requests.get(url, verify=False, headers=request_headers, params=())
    with open(file_name, "wb") as f:
        f.write(resp.content)

def extractTag(bs,tag):
    [s.extract() for s in bs(tag)]

def getToonKor( comicsUrl, baseUrl, baseDir):
    doc = getUrl(comicsUrl)
    title = doc.find("title").text
    table = doc.select("table.web_list")[0]
    elist = table.select("td.episode__index")

    new_dir = os.path.join(baseDir, title.replace(":","_"))
    if not os.path.isdir(new_dir): os.mkdir(new_dir)

    for e in elist:
        url = baseUrl + e['data-role']
        bs_img, html_img = getUrlHtml(url, request_headers)
        title = bs_img.find("title").text
        begin = html_img.index("var toon_img = '")
        end = html_img.index("';",begin)
        data = html_img[begin + 16: end]
        img_list = base64.b64decode(data.encode("UTF-8")).decode("UTF-8")
        doc_imgs = bs4.BeautifulSoup(img_list, 'html.parser')
        imgs = doc_imgs.select("img")
        
        sub_dir = os.path.join(new_dir, title.replace(":","_"))
        if not os.path.isdir(sub_dir): os.mkdir(sub_dir)

        k = 1;
        for img in imgs:
            img_url = img.get('src')
            if not img_url: continue
            if image_ext == None or img_url.endswith(image_ext):
                if( not img_url.startswith("http") ):
                    img_url = baseUrl + img_url
                file_name = "img_%04d.jpg" % k
                urlToFile( img_url, os.path.join( sub_dir, file_name) )
                print( img_url + " -> " + file_name )
                k = k + 1

if __name__ == "__main__":
        url = "https://tkor.lol/%EC%98%86%EC%A7%91%EC%86%8C%EB%85%80"
        iurl = "https://tkor.lol"
        bdir = "D:/Temp2/"
        getToonKor(url, iurl, bdir)