[Python] ToonKor Season 2 All

 ToonKor Season 2 All

import bs4, codecs
import requests
import base64
import os
import io

quit_flag = False

import signal
import sys

def signal_handler(sig, frame):
    quit_flag = True
    print('You pressed Ctrl+C!', quit_flag)    
    #sys.exit(0)

signal.signal(signal.SIGINT, signal_handler)
#print('Press Ctrl+C')
#signal.pause()


target_folder = r"D:/Temp6"

requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
image_ext = None
request_headers = {
    'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 '
                   '(KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'),
}

def safeFileName(filename):
    filename = filename.replace(":","_")
    filename = filename.replace("?","_")
    filename = filename.replace("/","_")
    filename = filename.replace("*","_")
    filename = filename.replace("<","_")
    filename = filename.replace(">","_")
    filename = filename.replace("\t","_")
    return filename.strip()
    
def getFile(url):
    with codecs.open(url,'r', encoding='utf8') as f:
        html = f.read()
    return bs4.BeautifulSoup(html, 'html.parser')
 
def getUrl(url, headers={}, params=()):
    resp = requests.get(url, verify=False, headers=headers, params=params)
    #resp.headers
    #html = resp.content.decode('utf8')
    html = resp.text
    return bs4.BeautifulSoup(html, 'html.parser')
 
def getUrlHtml(url, headers={}, params=()):
    resp = requests.get(url, verify=False, headers=headers, params=params)
    return bs4.BeautifulSoup(resp.text, 'html.parser'), resp.content.decode('utf8')
 
def urlToFile(url, file_name):
    resp = requests.get(url, verify=False, headers=request_headers, params=())
    with open(file_name, "wb") as f:
        f.write(resp.content)
 
def extractTag(bs,tag):
    [s.extract() for s in bs(tag)]
 
def getToonKor( comicsUrl, baseUrl, baseDir):
    while True:
        try:
            doc = getUrl(comicsUrl)  
            table = doc.select("table.bt_view2")[0]
            elist = table.select("td.bt_title")
            title = elist[0].text
            break
        except:
            print( comicsUrl, "-> retry")
            if quit_flag: return
            continue
 
    table = doc.select("table.web_list")[0]
    elist = table.select("td.content__title")
 
    new_dir = os.path.join(baseDir, safeFileName(title))

    if not os.path.isdir(new_dir): os.mkdir(new_dir)
    else: return
    
    count = 0
    for e in elist:
        count += 1
        url = baseUrl + e['data-role']
        title = e['alt']
        while True:
            try:
                bs_img, html_img = getUrlHtml(url, request_headers)
                begin = html_img.index("var tnimg = '")
                break
            except:
                print( url, "-> retry")
                if quit_flag: return
                continue
        end = html_img.index("';",begin)
        data = html_img[begin + 13: end]
        img_list = base64.b64decode(data.encode("UTF-8")).decode("UTF-8")
        doc_imgs = bs4.BeautifulSoup(img_list, 'html.parser')
        imgs = doc_imgs.select("img")
        
        #sub_dir = os.path.join(new_dir, title.replace(":","_"))
        #if not os.path.isdir(sub_dir): os.mkdir(sub_dir)
 
        html_file = os.path.join(new_dir, safeFileName(title) + ".html")
        
        if os.path.isfile(html_file): print(html_file, "-> exists"); continue
        print( len(elist), count, html_file)
        
        f = open( html_file, "w" )
        f.write('<meta name="referrer" content="no-referrer" /><br>\n')
        k = 1;
        for img in imgs:
            img_url = img.get('src')
            if not img_url: continue
            if image_ext == None or img_url.endswith(image_ext):
                if( not img_url.startswith("http") ):
                    img_url = baseUrl + img_url
                #file_name = "img_%04d.jpg" % k
                #urlToFile( img_url, os.path.join( sub_dir, file_name) )
                #print( img_url + " -> " + file_name )
                #print( img_url  )
                f.write('<img src="' + img_url + '" /><br>\n')
                k = k + 1
        f.close()

def saveToonKorComics():
    urls = [ 
        "https://tkr035.com/webtoon/1061",
        ]
    iurl = "https://tkr035.com"
    bdir = "D:/Temp2/"
    for url in urls:
        getToonKor(url, iurl, bdir)
        if quit_flag: break
    print("END")

def getToonKorList(list_url,start=0):
    doc = getUrl(list_url)
    lists = doc.select("div.section-item-inner")
    #print(lists)
    i = 0
    for l in lists:
        i += 1
        if i < start: continue
        comics = l.select("a")[0]
        print(i, len(lists), comics['alt'], comics['href'])
        getToonKor(comics['href'], "https://tkr035.com", target_folder)
        if quit_flag: break

def get_finished_webtoons():
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0", 36)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=2", 0)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=3", 0)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=4", 130)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=5", 0)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=6", 195)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=7", 0)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=8", 0)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=9", 0)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=10", 0)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=11", 0)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=12", 0)
    getToonKorList("https://tkr035.com/wt/%EC%99%84%EA%B2%B0?gbun=&wpage=&page=13", 0)

def get_continue_webtoons():
    getToonKorList("https://tkr035.com/wt/%EC%B5%9C%EC%8B%A0/0/all/%EC%9D%B8%EA%B8%B0//%EC%A0%84%EC%B2%B4", 0)

def get_week_webtoons(week,page,start=0):
    global target_folder
    target_folder = r"D:/Temp7/" + str(week)
    url = "https://tkr035.com/wt/%EC%97%B0%EC%9E%AC%EC%A4%91/"
    url += str(week)
    url += "/all/%EC%9D%B8%EA%B8%B0/%EC%A0%84%EC%B2%B4?gbun=&wpage=&page="
    url += str(page)
    getToonKorList(url,start)
    
def get_week_webtoons_all():
    #for i in range(3): get_week_webtoons(1,i+1)
    #get_week_webtoons(1,2,150)
    #get_week_webtoons(1,3)
    for i in range(3): get_week_webtoons(2,i+1)
    for i in range(3): get_week_webtoons(3,i+1)
    for i in range(3): get_week_webtoons(4,i+1)
    for i in range(3): get_week_webtoons(5,i+1)
    for i in range(3): get_week_webtoons(6,i+1)
    for i in range(3): get_week_webtoons(7,i+1)
    for i in range(1): get_week_webtoons(8,i+1)
        
    '''
    global target_folder
    target_folder = r"D:/Temp7/1"
    getToonKorList("https://tkr035.com/wt/%EC%97%B0%EC%9E%AC%EC%A4%91/1/all/%EC%9D%B8%EA%B8%B0/%EC%A0%84%EC%B2%B4")
    getToonKorList("https://tkr035.com/wt/%EC%97%B0%EC%9E%AC%EC%A4%91/1/all/%EC%9D%B8%EA%B8%B0/%EC%A0%84%EC%B2%B4?gbun=&wpage=&page=2")
    getToonKorList("https://tkr035.com/wt/%EC%97%B0%EC%9E%AC%EC%A4%91/1/all/%EC%9D%B8%EA%B8%B0/%EC%A0%84%EC%B2%B4?gbun=&wpage=&page=3")
    '''

if __name__ == "__main__":
    #get_continue_webtoons()
    get_week_webtoons_all()
    
    

[Python] ToonKor Season 2

 ToonKor Season 2

import bs4, codecs
import requests
import base64
import os
import io
import sys
 
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
image_ext = None
request_headers = {
    'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 '
                   '(KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'),
}

def safeFileName(filename):
    return filename.replace(":","_").replace("?","_").replace("<","_").replace(">","_").strip()

def getFile(url):
    with codecs.open(url,'r', encoding='utf8') as f:
        html = f.read()
    return bs4.BeautifulSoup(html, 'html.parser')
 
def getUrl(url, headers={}, params=()):
    resp = requests.get(url, verify=False, headers=headers, params=params)
    #resp.headers
    #html = resp.content.decode('utf8')
    html = resp.text
    return bs4.BeautifulSoup(html, 'html.parser')
 
def getUrlHtml(url, headers={}, params=()):
    resp = requests.get(url, verify=False, headers=headers, params=params)
    return bs4.BeautifulSoup(resp.text, 'html.parser'), resp.content.decode('utf8')
 
def urlToFile(url, file_name):
    resp = requests.get(url, verify=False, headers=request_headers, params=())
    with open(file_name, "wb") as f:
        f.write(resp.content)
 
def extractTag(bs,tag):
    [s.extract() for s in bs(tag)]
 
def getToonKor( comicsUrl, baseUrl, baseDir):
    while True:
        try:
            doc = getUrl(comicsUrl)
            table = doc.select("table.bt_view2")[0]
            elist = table.select("td.bt_title")
            title = elist[0].text
            break
        except:
            print(comicsUrl, "-> retry")
            continue
 
    table = doc.select("table.web_list")[0]
    elist = table.select("td.content__title")
 
    new_dir = os.path.join(baseDir, safeFileName(title))
    if not os.path.isdir(new_dir): os.mkdir(new_dir)
 
    for e in elist:
        url = baseUrl + e['data-role']
        title = e['alt']
        while True:
            try:
                bs_img, html_img = getUrlHtml(url, request_headers)
                begin = html_img.index("var tnimg = '")
                break
            except:
                print( url, "-> retry")
                continue
        end = html_img.index("';",begin)
        data = html_img[begin + 13: end]
        img_list = base64.b64decode(data.encode("UTF-8")).decode("UTF-8")
        doc_imgs = bs4.BeautifulSoup(img_list, 'html.parser')
        imgs = doc_imgs.select("img")
        
        #sub_dir = os.path.join(new_dir, title.replace(":","_"))
        #if not os.path.isdir(sub_dir): os.mkdir(sub_dir)
 
        html_file = os.path.join(new_dir, safeFileName(title) + ".html")
        print(html_file)
        f = open( html_file, "w" )
        f.write('<meta name="referrer" content="no-referrer" /><br>\n')
        k = 1;
        for img in imgs:
            img_url = img.get('src')
            if not img_url: continue
            if image_ext == None or img_url.endswith(image_ext):
                if( not img_url.startswith("http") ):
                    img_url = baseUrl + img_url
                #file_name = "img_%04d.jpg" % k
                #urlToFile( img_url, os.path.join( sub_dir, file_name) )
                #print( img_url + " -> " + file_name )
                print( img_url  )
                f.write('<img src="' + img_url + '" /><br>\n')
                k = k + 1
        f.close()
        
if __name__ == "__main__":
    urls = []
    if len(sys.argv) > 1:
        for i in range(1,len(sys.argv)):
            urls.append(sys.argv[i])
    else:
        urls.append(
            #"https://tkr035.com/webtoon/2939" #사내맞선
            #"https://tkr035.com/webtoon/826" #황제의 외동딸
            #"https://tkr035.com/webtoon/2794" #그만바의 자취방
            #"https://tkr035.com/webtoon/2647" #첩
            "https://tkr035.com/webtoon/6117" #외모지상주의
        )
    iurl = "https://tkr035.com"
    bdir = "D:/Temp2/"
    for url in urls:
        getToonKor(url, iurl, bdir)
    print("END")

[python] ToonKor Season 1

 ToonKor Season 1

import bs4, codecs
import requests
import base64
import os
import io
import sys
import re

requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
image_ext = None
request_headers = {
    'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 '
                   '(KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'),
}

def safeFileName(filename):
    return re.sub("[:?/*<>\t.]", "_", filename ).strip()
    
def getFile(url):
    with codecs.open(url,'r', encoding='utf8') as f:
        html = f.read()
    return bs4.BeautifulSoup(html, 'html.parser')

def getUrl(url, headers={}, params=()):
    resp = requests.get(url, verify=False, headers=headers, params=params)
    #resp.headers
    #html = resp.content.decode('utf8')
    html = resp.text
    return bs4.BeautifulSoup(html, 'html.parser')

def getUrlHtml(url, headers={}, params=()):
    resp = requests.get(url, verify=False, headers=headers, params=params)
    return bs4.BeautifulSoup(resp.text, 'html.parser'), resp.content.decode('utf8')

def urlToFile(url, file_name):
    resp = requests.get(url, verify=False, headers=request_headers, params=())
    with open(file_name, "wb") as f:
        f.write(resp.content)

def extractTag(bs,tag):
    [s.extract() for s in bs(tag)]

def getToonKor( comicsUrl, baseUrl, baseDir):
    doc = getUrl(comicsUrl)
    title = doc.find("title").text
    table = doc.select("table.web_list")[0]
    elist = table.select("td.episode__index")

    #print( doc, title, table, elist )
    
    new_dir = os.path.join(baseDir, safeFileName(title))
    if not os.path.isdir(new_dir): os.mkdir(new_dir)

    for e in elist:
        url = baseUrl + e['data-role']
        bs_img, html_img = getUrlHtml(url, request_headers)
        title = bs_img.find("title").text
        begin = html_img.index("var toon_img = '")
        end = html_img.index("';",begin)
        data = html_img[begin + 16: end]
        img_list = base64.b64decode(data.encode("UTF-8")).decode("UTF-8")
        doc_imgs = bs4.BeautifulSoup(img_list, 'html.parser')
        imgs = doc_imgs.select("img")
        
        sub_dir = os.path.join(new_dir, safeFileName(title))
        if not os.path.isdir(sub_dir): os.mkdir(sub_dir)
        else: print( 'skip -->', sub_dir ); continue
        print(sub_dir)

        k = 1;
        for img in imgs:
            img_url = img.get('src')
            #print(img_url)
            if not img_url: continue
            if image_ext == None or img_url.endswith(image_ext):
                if( not img_url.startswith("http") ):
                    img_url = baseUrl + img_url
                ext = img_url.rfind(".")
                if ext >= 0: file_name = ("img_%04d" % k) + img_url[ext:]
                else: file_name = "img_%04d.jpg" % k
                urlToFile( img_url, os.path.join( sub_dir, file_name) )
                print( img_url + " -> " + file_name )
                k = k + 1

if __name__ == "__main__":
    #https://tkor.fish/%EC%9B%B9%ED%88%B0
    url = "https://tkor.fish/%EB%82%A8%EC%B9%9C-%EC%97%86%EC%9D%8C-%EB%82%A8%EC%B9%9C-%EC%9E%88%EC%9D%8C"
    baseUrl = "https://tkor.fish"
    outDir = "D:/Temp2/"
    if len(sys.argv) > 1:
        url = sys.argv[1]    
        baseUrl = url[:url.find('/',8)]
    if len(sys.argv) > 2:
        outDir = sys.argv[2]
    getToonKor(url, baseUrl, outDir)
    
        

[Python| Markdown file merge

Markdown File Merge

import os
import sys

def text_merge(folder):
    files = sorted(os.listdir(folder))
    with open( folder + ".md", "w", encoding="utf-8" ) as w:
        for f in files:
            path = os.path.join(folder,f)
            with open( path, "r", encoding='utf-8') as r:
                w.write(r.read())
                w.write('\n\n<div style="page-break-after: always;"></div><br>\n\n')
                print(path)
        
if __name__ == "__main__":
    if len(sys.argv) > 1:
        text_merge(sys.argv[1])

[Python] Any to UTF-8

 Any codec to UTF-8

#-*- coding: utf-8 -*-

import sys    
import os

def file_enc(path):
    import chardet
    with open( path, "rb" ) as f:
        return chardet.detect(f.read()).get('encoding')
    
def py2_euc2utf(in_file, out_file):
    with open(in_file, "r") as f:
        euc = f.read().decode('cp949') #encode('cp949').decode('cp437')
        #euc = f.read().decode('euc_kr') #encode('cp949').decode('cp437')
        utf = euc.encode('utf-8')
        with open(out_file, "w") as w:
            w.write(utf)

def py3_euc2utf(in_file, out_file):
    with open(in_file, "r", encoding=file_enc(in_file)) as f:
        utf = f.read()
        with open(out_file, "w", encoding="utf-8") as w:
            w.write(utf)

def euc2utf(in_file, out_file):
    if sys.version_info.major == 3: py3_euc2utf(in_file,  out_file + '.py3')
    else: py2_euc2utf(in_file, out_file + '.py2')

    
def filelist(path):
    from os import listdir
    from os.path import isfile, join
    return [f for f in listdir(path) if isfile(join(path, f))]

if __name__ == "__main__":
    in_dir = r"d:/Ebook/가림토txt"
    out_dir = r"d:Ebook/가림토txt_utf"
    if not os.path.exists(out_dir):
        os.mkdir(out_dir)
    files = filelist(in_dir)
    for f in files:
        print(f, file_enc(os.path.join(in_dir,f)))
        try:
            euc2utf(os.path.join(in_dir,f),os.path.join(out_dir,f))
        except:
            print(f, "--------------> ERROR")
    

[python] TookKor example (jython)

 TookKor Jython Example

# -*- coding: utf-8
 
import os
import sys
sys.path.append(r"d:\Lib\jar\jsoup-1.12.1.jar")

from java.io import *
from java.net import *
from java.util import *
from java.lang import *
from java.nio.file import *

from org.jsoup import *
from org.jsoup.nodes import *
from org.jsoup.select import *

debug = True
filter = None #"jpg"

def decodeBase64(data):
    import base64
    try:
        return base64.b64decode(data)
    except BaseException as e:
        print('except', str(e))
        return None

def urlToFile(urlStr, fileName, referer):
    url = URL(urlStr)
    hc  = url.openConnection()
    hc.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36")
    hc.setRequestProperty("Referer", referer);
       
    status = hc.getResponseCode();
    while status != HttpURLConnection.HTTP_OK: # and status != HttpURLConnection.HTTP_NOT_FOUND:
        if status == HttpURLConnection.HTTP_MOVED_TEMP or status == HttpURLConnection.HTTP_MOVED_PERM or status == HttpURLConnection.HTTP_SEE_OTHER:
            newUrl = hc.getHeaderField("Location")
            hc = URL(newUrl).openConnection()
            hc.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36");
            status = hc.getResponseCode();           
    Files.copy(hc.getInputStream(), Paths.get(fileName), StandardCopyOption.REPLACE_EXISTING)
    
def getJsoupDocument(url):
    while True:
        try:
            return Jsoup.connect(url).get()
        except:
            Thread.sleep(1000)

def getToonKor(comicsUrl, baseUrl, baseDir):
    doc_toc = getJsoupDocument(comicsUrl)
    #if debug: print(doc_toc.html()) 

    table = doc_toc.select("table[class=web_list]").first()
    list = table.select("td[class=episode__index]")
    #if debug: print(list.html())

    dir = os.path.join(baseDir,doc_toc.title().replace(":","_"))  
    if not os.path.isdir(dir): os.mkdir(dir) #dir.mkdirs()

    for e in list:
        #if( i++ < 38 ) continue;
        try:
            url = baseUrl + e.attr("data-role")
            if debug: print(url)
    
            doc_img = getJsoupDocument(url)
            html_img = doc_img.html()
            begin = html_img.find("var toon_img = '");
            end = html_img.find("';",begin);
            data = html_img[begin+16:end]
            
            img_list = decodeBase64(data)
            doc_imgs = Jsoup.parse(img_list)
            imgs = doc_imgs.select("img")

            subdir = os.path.join(dir,doc_img.title().replace(":","_"))
            if not os.path.isdir(subdir): os.mkdir(subdir) 
    
            k = 1;
            for img in imgs:
                img_url = img.attr("src")
                print(img_url)
                if filter == None or img_url.endswith(filter):
                    if not img_url.startswith("http"):
                        img_url = baseUrl + img_url;
                    file_name = "img_%04d.jpg" % k
                    k = k + 1
                    print( img_url + " -> " + file_name )
                    urlToFile(img_url, os.path.join(subdir, file_name), comicsUrl)
        except BaseException as e:
            print('except2', str(e))

def getWolfCom(comicsUrl, baseUrl, baseDir):
    doc_toc = getJsoupDocument(comicsUrl)
    if debug: print(doc_toc.html()) #doc_toc.text()

    list = doc_toc.select("div.box > div.group.left-box > div.webtoon-bbs-list.bbs-list > ul > li")
    if debug: print(list.html())

    dir = os.path.join(baseDir,doc_toc.title().replace(":","_"))  
    if not os.path.isdir(dir): os.mkdir(dir) #dir.mkdirs()

    for e in list:
        #if( i++ < 38 ) continue;
        try:
            url = baseUrl + e.select("a").first().attr("href")
            if debug: print(url)
    
            doc_img = getJsoupDocument(url)
            imgs = doc_img.select("section.webtoon-body > div.group.image-view > img")
            print(doc_img.title())
            
            subdir = os.path.join(dir,doc_img.title().replace(":","_"))
            if not os.path.isdir(subdir): os.mkdir(subdir) #subdir.mkdirs()
    
            k = 1;
            for img in imgs:
                img_url = img.attr("src")
                if filter == None or img_url.endswith(filter):
                    if not img_url.startswith("http"):
                        img_url = baseUrl + img_url;
                    file_name = "img_%04d.jpg" % k
                    k = k + 1
                    print( img_url + " -> " + file_name )
                    urlToFile(img_url, os.path.join(subdir, file_name), comicsUrl)
        except:
            pass

if __name__ == "__main__":
    url  = "https://tkor.work/%EB%AA%A8%EA%B8%B0%EB%96%BC"
    iurl = "https://tkor.work"
    dir  = "D:/Temp2/"
    getToonKor(url, iurl, dir)
    print("END")

[python] ebook maker

 

ebook maker

import os
import sys
import codecs
import bs4

class html():
    def __init__(self,filename=None):
        self.html = None
        self.bs = None
        if filename:
            self.open(filename)
    def open(self,filename):
        with codecs.open(filename,'r', encoding='utf8') as f:
            self.html = f.read()
        self.bs = bs4.BeautifulSoup(self.html, 'html.parser')  
    def tags(self,tag,attrs=None): #tag='div', attrs = { 'id' : id }
        if attrs == None:
            return self.bs.find_all(tag)
        else:
            return self.bs.find_all(tag,attrs)
    def tag(self,tag): # a.class, a#id, a tag
        return self.bs.select(tag)
    def remove(self,css):
        tags = self.bs.select(css)
        if tags:
            for tag in tags:
                tag.extract()
    def removeTags(self,tag):
        return [s.extract() for s in self.bs(tag)]
    def save(self,filename):
        with codecs.open(filename,'w', encoding='utf8') as f:
            f.write(self.bs.decode_contents())
            f#.write(self.bs.html.decode_contents()) #get_text() or prettify()
    def appendHead(self,tag,text,attrs):
        new_tag = self.bs.new_tag(tag, **attrs) #soup.new_tag('div', **{"data-role": "content"})
        if text: new_tag.string = text
        self.bs.head.append(new_tag)
    def wrapBody(self,tag,attrs):
        wrapper = self.bs.new_tag(tag, **attrs) #soup.new_tag('div', **{"data-role": "content"})
        body_children = list(self.bs.body.children)
        self.bs.body.clear()
        self.bs.body.append(wrapper)
        for child in body_children:
            wrapper.append(child)
            
def files(dirname):
    fs = []
    files = os.listdir(dirname)
    for f in files:
        name,ext = os.path.splitext(f)
        if ext == '.html':
            fs.append(os.path.join(dirname,f))
    return fs
    
def stripTags():
    fs = files(r'D:/Temp2/4')
    for f in fs:
        print(f)
        h = html(f)
        h.remove('iframe')
        h.remove('script')
        h.remove('div.navbar')
        h.remove('div.articleTopAd')
        h.remove('li.previous')
        h.remove('li.next')
        h.remove('div#footerDiv')
        h.remove('nav.col-md-3')
        h.save(f + '.htm')
        
def insertEbookTags(f):
    h = html(f)

    h.appendHead('link', None, { 'href': '../css/bootstrap.min.css', 'rel': 'stylesheet' })
    h.appendHead('link', None, { 'href': '../css/styles.css', 'rel': 'stylesheet' })
    h.appendHead('link', None, { 'href': '../css/highlight.pack.css', 'rel': 'stylesheet' })
    h.appendHead('script', None, { 'src': '../js/highlight.pack.js' })
    h.appendHead('script', r'hljs.configure({tabReplace: '    '}); hljs.initHighlightingOnLoad();', {})
    
    h.wrapBody('div', { 'class': 'main' } )
    h.wrapBody('div', { 'class': 'col-xs-12' } )
    h.wrapBody('div', { 'class': 'row' } )
    h.wrapBody('div', { 'class': 'container-fluid' } )

    h.save(f + '.htm')
            
def makeEbook(directory):
    fs = files(directory)
    for f in fs:
        insertEbookTags(f)
        print(f)

def getLinks(filename):
    h = html(filename)
    data = [ ]
    links = h.tags('a');
    for l in links:
        if not l.attrs:
            continue
        try:
            href = l.attrs['href']
            data.append( '{ "' + l.text.replace('[\r\n]','') + '", "' + href + '" },\n')
        except:
            pass
    with codecs.open(filename + '.txt','w','utf-8') as f:
        for d in data:
            f.write(d)
        
if __name__ == '__main__':
    #stripTags()
    insertEbookTags(r'd:\Android\ebook\work\expect\autoexpect.html')
    #makeEbook('d:\\Android\\ebook\\work\\advbash')
    #getLinks('d:\\Android\\ebook\\work\\advbash\\index.html')
    #getLinks('d:\\Android\\ebook\\bs4_index.html')