from __future__ import absolute_import
from docstructure import SITE_STRUCTURE, HREF_MAP, BASENAME_MAP
from lxml.etree import (parse, fromstring, ElementTree,
Element, SubElement, XPath, XML)
import os
import re
import sys
import copy
import shutil
import textwrap
import subprocess
try:
from io import open as open_file
except ImportError:
from codecs import open as open_file
RST2HTML_OPTIONS = " ".join([
'--no-toc-backlinks',
'--strip-comments',
'--language en',
'--date',
])
XHTML_NS = 'http://www.w3.org/1999/xhtml'
htmlnsmap = {"h": XHTML_NS}
find_head = XPath("/h:html/h:head[1]", namespaces=htmlnsmap)
find_title = XPath("/h:html/h:head/h:title/text()", namespaces=htmlnsmap)
find_title_tag = XPath("/h:html/h:head/h:title", namespaces=htmlnsmap)
find_headings = XPath("//h:h1[not(@class)]//text()", namespaces=htmlnsmap)
find_heading_tag = XPath("//h:h1[@class = 'title'][1]", namespaces=htmlnsmap)
find_menu = XPath("//h:ul[@id=$name]", namespaces=htmlnsmap)
find_page_end = XPath("/h:html/h:body/h:div[last()]", namespaces=htmlnsmap)
find_words = re.compile('(\w+)').findall
replace_invalid = re.compile(r'[-_/.\s\\]').sub
def make_menu_section_head(section, menuroot):
section_id = section + '-section'
section_head = menuroot.xpath("//ul[@id=$section]/li", section=section_id)
if not section_head:
ul = SubElement(menuroot, "ul", id=section_id)
section_head = SubElement(ul, "li")
title = SubElement(section_head, "span", {"class":"section title"})
title.text = section
else:
section_head = section_head[0]
return section_head
def build_menu(tree, basename, section_head):
page_title = find_title(tree)
if page_title:
page_title = page_title[0]
else:
page_title = replace_invalid('', basename.capitalize())
build_menu_entry(page_title, basename+".html", section_head,
headings=find_headings(tree))
def build_menu_entry(page_title, url, section_head, headings=None):
page_id = replace_invalid(' ', os.path.splitext(url)[0]) + '-menu'
ul = SubElement(section_head, "ul", {"class":"menu foreign", "id":page_id})
title = SubElement(ul, "li", {"class":"menu title"})
a = SubElement(title, "a", href=url)
a.text = page_title
if headings:
subul = SubElement(title, "ul", {"class":"submenu"})
for heading in headings:
li = SubElement(subul, "li", {"class":"menu item"})
try:
ref = heading.getparent().getparent().get('id')
except AttributeError:
ref = None
if ref is None:
ref = '-'.join(find_words(replace_invalid(' ', heading.lower())))
a = SubElement(li, "a", href=url+'#'+ref)
a.text = heading
def merge_menu(tree, menu, name):
menu_root = copy.deepcopy(menu)
tree.getroot()[1][0].insert(0, menu_root) # html->body->div[class=document]
for el in menu_root.iter():
tag = el.tag
if tag[0] != '{':
el.tag = "{http://www.w3.org/1999/xhtml}" + tag
current_menu = find_menu(
menu_root, name=replace_invalid(' ', name) + '-menu')
if not current_menu:
current_menu = find_menu(
menu_root, name=replace_invalid('-', name) + '-menu')
if current_menu:
for submenu in current_menu:
submenu.set("class", submenu.get("class", "").
replace("foreign", "current"))
return tree
def inject_flatter_button(tree):
head = tree.xpath('h:head[1]', namespaces=htmlnsmap)[0]
script = SubElement(head, '{%s}script' % XHTML_NS, type='text/javascript')
script.text = """
(function() {
var s = document.createElement('script');
var t = document.getElementsByTagName('script')[0];
s.type = 'text/javascript';
s.async = true;
s.src = 'http://api.flattr.com/js/0.6/load.js?mode=auto';
t.parentNode.insertBefore(s, t);
})();
"""
script.tail = '\n'
intro_div = tree.xpath('h:body//h:div[@id = "introduction"][1]', namespaces=htmlnsmap)[0]
intro_div.insert(-1, XML(
'
Like working with lxml? '
'Happy about the time that it just saved you?
'
'Show your appreciation with Flattr.
'
''
'
'
))
def inject_donate_buttons(lxml_path, rst2html_script, tree):
command = ([sys.executable, rst2html_script]
+ RST2HTML_OPTIONS.split() + [os.path.join(lxml_path, 'README.rst')])
rst2html = subprocess.Popen(command, stdout=subprocess.PIPE)
stdout, _ = rst2html.communicate()
readme = fromstring(stdout)
intro_div = tree.xpath('h:body//h:div[@id = "introduction"][1]',
namespaces=htmlnsmap)[0]
support_div = readme.xpath('h:body//h:div[@id = "support-the-project"][1]',
namespaces=htmlnsmap)[0]
intro_div.append(support_div)
legal = readme.xpath('h:body//h:div[@id = "legal-notice-for-donations"][1]',
namespaces=htmlnsmap)[0]
last_div = tree.xpath('h:body//h:div//h:div', namespaces=htmlnsmap)[-1]
last_div.addnext(legal)
def rest2html(script, source_path, dest_path, stylesheet_url):
command = ('%s %s %s --stylesheet=%s --link-stylesheet %s > %s' %
(sys.executable, script, RST2HTML_OPTIONS,
stylesheet_url, source_path, dest_path))
subprocess.call(command, shell=True)
def convert_changelog(lxml_path, changelog_file_path, rst2html_script, stylesheet_url):
f = open_file(os.path.join(lxml_path, 'CHANGES.txt'), 'r', encoding='utf-8')
try:
content = f.read()
finally:
f.close()
links = dict(LP='`%s `_',
GH='`%s `_')
replace_tracker_links = re.compile('((LP|GH)#([0-9]+))').sub
def insert_link(match):
text, ref_type, ref_id = match.groups()
return links[ref_type] % (text, ref_id)
content = replace_tracker_links(insert_link, content)
command = [sys.executable, rst2html_script] + RST2HTML_OPTIONS.split() + [
'--link-stylesheet', '--stylesheet', stylesheet_url ]
out_file = open(changelog_file_path, 'wb')
try:
rst2html = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=out_file)
rst2html.communicate(content.encode('utf8'))
finally:
out_file.close()
def publish(dirname, lxml_path, release):
if not os.path.exists(dirname):
os.mkdir(dirname)
doc_dir = os.path.join(lxml_path, 'doc')
script = os.path.join(doc_dir, 'rest2html.py')
pubkey = os.path.join(doc_dir, 'pubkey.asc')
stylesheet_url = 'style.css'
shutil.copy(pubkey, dirname)
href_map = HREF_MAP.copy()
changelog_basename = 'changes-%s' % release
href_map['Release Changelog'] = changelog_basename + '.html'
menu_js = textwrap.dedent('''
function trigger_menu() {
var sidemenu = document.getElementById("sidemenu");
var classes = sidemenu.getAttribute("class");
if (classes.indexOf(" visible") === -1) {
sidemenu.setAttribute("class", classes + " visible");
} else {
sidemenu.setAttribute("class", classes.replace(" visible", ""));
}
}
''')
trees = {}
menu = Element("div", {'class': 'sidemenu', 'id': 'sidemenu'})
SubElement(SubElement(menu, 'div', {'class': 'menutrigger', 'onclick': 'trigger_menu()'}), 'a').text = "Menu"
menu_div = SubElement(menu, 'div', {'class': 'menu'})
# build HTML pages and parse them back
for section, text_files in SITE_STRUCTURE:
section_head = make_menu_section_head(section, menu_div)
for filename in text_files:
if filename.startswith('@'):
# special menu entry
page_title = filename[1:]
url = href_map[page_title]
build_menu_entry(page_title, url, section_head)
else:
path = os.path.join(doc_dir, filename)
basename = os.path.splitext(os.path.basename(filename))[0]
basename = BASENAME_MAP.get(basename, basename)
outname = basename + '.html'
outpath = os.path.join(dirname, outname)
rest2html(script, path, outpath, stylesheet_url)
tree = parse(outpath)
if filename == 'main.txt':
# inject donation buttons
#inject_flatter_button(tree)
inject_donate_buttons(lxml_path, script, tree)
trees[filename] = (tree, basename, outpath)
build_menu(tree, basename, section_head)
# also convert CHANGES.txt
convert_changelog(lxml_path, os.path.join(dirname, 'changes-%s.html' % release),
script, stylesheet_url)
# generate sitemap from menu
sitemap = XML(textwrap.dedent('''\
Sitemap of lxml.de - Processing XML and HTML with Python
Sitemap of lxml.de - Processing XML and HTML with Python
'''))
sitemap_menu = copy.deepcopy(menu)
SubElement(SubElement(sitemap_menu[-1], 'li'), 'a', href='http://lxml.de/files/').text = 'Download files'
sitemap[-1].append(sitemap_menu) # append to body
ElementTree(sitemap).write(os.path.join(dirname, 'sitemap.html'))
# integrate sitemap into the menu
SubElement(SubElement(menu_div[-1], 'li'), 'a', href='http://lxml.de/sitemap.html').text = 'Sitemap'
# integrate menu into web pages
for tree, basename, outpath in trees.itervalues():
head = find_head(tree)[0]
SubElement(head, 'script', type='text/javascript').text = menu_js
new_tree = merge_menu(tree, menu, basename)
title = find_title_tag(new_tree)
if title and title[0].text == 'lxml':
title[0].text = "lxml - Processing XML and HTML with Python"
heading = find_heading_tag(new_tree)
if heading:
heading[0].text = "lxml - XML and HTML with Python"
new_tree.write(outpath)
if __name__ == '__main__':
publish(sys.argv[1], sys.argv[2], sys.argv[3])