Move Markdown handling to the translation classes

- the Session has a `pokedex_link_maker` property, whose `object_url`
  method is used to make URLs in Markdown
- pokemon.names_table.name is now an ordinary Unicode column
- pokemon.name is a MarkdownString that is aware of the session and the
  language the string is in
- pokemon.name_map is a dict-like association_proxy of the above
- move.effect works similarly, with transparent $effect_chance substitution
  as before
This commit is contained in:
Petr Viktorin 2011-04-20 21:47:37 +02:00
parent 5a6ff3d27b
commit 85d779ba83
5 changed files with 165 additions and 119 deletions

View file

@ -5,9 +5,9 @@ The language used is a variation of Markdown and Markdown Extra. There are
docs for each at http://daringfireball.net/projects/markdown/ and docs for each at http://daringfireball.net/projects/markdown/ and
http://michelf.com/projects/php-markdown/extra/ respectively. http://michelf.com/projects/php-markdown/extra/ respectively.
Pokédex links are represented with the syntax `[text]{type:identifier}`, e.g., Pokédex links are represented with the syntax `[label]{category:identifier}`,
`[Eevee]{pokemon:eevee}`. The actual code that parses these is in e.g., `[Eevee]{pokemon:eevee}`. The label can (and should) be left out, in
spline-pokedex. which case it is replaced by the name of the thing linked to.
""" """
from __future__ import absolute_import from __future__ import absolute_import
@ -15,40 +15,52 @@ import re
import markdown import markdown
import sqlalchemy.types import sqlalchemy.types
from sqlalchemy.orm.session import object_session
class MarkdownString(object): class MarkdownString(object):
"""Wraps a Markdown string. Stringifies to the original text, but .as_html """Wraps a Markdown string.
will return an HTML rendering.
To make the __html__ property work, you must set this class's Use unicode() and __html__ for text and HTML representations.
`default_link_extension` to a PokedexLinkExtension. Yep, that's gross. The as_text() and as_html() functions do the same, but accept optional
arguments that may affect the rendering.
The `source_text` property holds the original text.
init args:
`source_text`: the text in Markdown syntax
`session`: A DB session used for looking up linked objects
`language`: The language the string is in. If None, the session default
is used.
""" """
default_link_extension = None default_link_extension = None
def __init__(self, source_text): def __init__(self, source_text, session, language):
self.source_text = source_text self.source_text = source_text
self.session = session
self.language = language
def __unicode__(self): def __unicode__(self):
return self.source_text return self.as_text()
def __str__(self): def __str__(self):
return unicode(self.source_text).encode() return self.as_text().encode()
def __html__(self): def __html__(self):
return self.as_html(extension=self.default_link_extension) return self.as_html()
def as_html(self, session=None, object_url=None, identifier_url=None, language=None, extension=None): def as_html(self, object_url=None, identifier_url=None, make_link=None):
"""Returns the string as HTML. """Returns the string as HTML.
Pass in current session, and optionally URL-making functions and the If given, the optional arguments will be used instead of those in the
language. See PokedexLinkExtension for how they work. session's pokedex_link_maker. See MarkdownLinkMaker for documentation.
Alternatively, pass in a PokedexLinkExtension instance.
""" """
if not extension: extension = self.session.pokedex_link_maker.get_extension(
extension = ParametrizedLinkExtension(session, object_url, identifier_url, language) self.language,
object_url=object_url,
identifier_url=identifier_url,
make_link=make_link,
)
md = markdown.Markdown( md = markdown.Markdown(
extensions=['extra', extension], extensions=['extra', extension],
@ -58,20 +70,26 @@ class MarkdownString(object):
return md.convert(self.source_text) return md.convert(self.source_text)
def as_text(self, session): def as_text(self):
"""Returns the string in a plaintext-friendly form. """Returns the string in a plaintext-friendly form.
Currently there are no tunable parameters
""" """
# Since Markdown is pretty readable by itself, we just have to replace # Since Markdown is pretty readable by itself, we just have to replace
# the links by their text. # the links by their text.
# XXX: The tables get unaligned # XXX: The tables get unaligned
extension = ParametrizedLinkExtension(session)
pattern = extension.link_pattern link_maker = MarkdownLinkMaker(self.session)
pattern = PokedexLinkPattern(link_maker, self.language)
regex = '()%s()' % pattern.regex regex = '()%s()' % pattern.regex
def handleMatch(m): def handleMatch(m):
return pattern.handleMatch(m).text return pattern.handleMatch(m).text
return re.sub(regex, handleMatch, self.source_text) return re.sub(regex, handleMatch, self.source_text)
def _markdownify_effect_text(move, effect_text): def _markdownify_effect_text(move, effect_text, language=None):
session = object_session(move)
if effect_text is None: if effect_text is None:
return effect_text return effect_text
effect_text = effect_text.replace( effect_text = effect_text.replace(
@ -79,7 +97,7 @@ def _markdownify_effect_text(move, effect_text):
unicode(move.effect_chance), unicode(move.effect_chance),
) )
return MarkdownString(effect_text) return MarkdownString(effect_text, session, language)
class MoveEffectProperty(object): class MoveEffectProperty(object):
"""Property that wraps move effects. Used like this: """Property that wraps move effects. Used like this:
@ -110,42 +128,22 @@ class MoveEffectPropertyMap(MoveEffectProperty):
prop = getattr(obj.move_effect, self.effect_column) prop = getattr(obj.move_effect, self.effect_column)
newdict = dict(prop) newdict = dict(prop)
for key in newdict: for key in newdict:
newdict[key] = _markdownify_effect_text(obj, newdict[key]) newdict[key] = _markdownify_effect_text(obj, newdict[key], key)
return newdict return newdict
class MarkdownColumn(sqlalchemy.types.TypeDecorator):
"""Generic SQLAlchemy column type for Markdown text.
Do NOT use this for move effects! They need to know what move they belong
to so they can fill in, e.g., effect chances. Use the MoveEffectProperty
property class above.
"""
impl = sqlalchemy.types.Unicode
def process_bind_param(self, value, dialect):
if value is None:
return None
if not isinstance(value, basestring):
# Can't assign, e.g., MarkdownString objects yet
raise NotImplementedError
return unicode(value)
def process_result_value(self, value, dialect):
if value is None:
return None
return MarkdownString(value)
class PokedexLinkPattern(markdown.inlinepatterns.Pattern): class PokedexLinkPattern(markdown.inlinepatterns.Pattern):
"""Matches [label]{category:target}. """Matches [label]{category:target}.
Handles matches using factory
""" """
regex = ur'(?x) \[ ([^]]*) \] \{ ([-a-z0-9]+) : ([-a-z0-9]+) \}' regex = ur'(?x) \[ ([^]]*) \] \{ ([-a-z0-9]+) : ([-a-z0-9]+) \}'
def __init__(self, extension): def __init__(self, factory, string_language, game_language=None):
markdown.inlinepatterns.Pattern.__init__(self, self.regex) markdown.inlinepatterns.Pattern.__init__(self, self.regex)
self.extension = extension self.factory = factory
self.session = factory.session
self.string_language = string_language
self.game_language = game_language
def handleMatch(self, m): def handleMatch(self, m):
from pokedex.db import tables, util from pokedex.db import tables, util
@ -161,47 +159,54 @@ class PokedexLinkPattern(markdown.inlinepatterns.Pattern):
)[category] )[category]
except KeyError: except KeyError:
obj = name = target obj = name = target
url = self.extension.identifier_url(category, obj) url = self.factory.identifier_url(category, obj)
else: else:
session = self.extension.session session = self.session
obj = util.get(self.extension.session, table, target) obj = util.get(self.session, table, target)
url = self.extension.object_url(category, obj) url = self.factory.object_url(category, obj)
url = url or self.factory.identifier_url(category, obj.identifier)
name = None
# Translations can be incomplete; in which case we want to use a
# fallback.
if table in [tables.Type]: if table in [tables.Type]:
# Type wants to be localized to the same language as the text # Type wants to be localized to the same language as the text
language = self.extension.language name = obj.name_map.get(self.string_language)
name = None if not name and self.game_language:
try: name = obj.name_map.get(self.game_language)
name = obj.name_map[language]
except KeyError:
pass
if not name: if not name:
name = obj.name name = obj.name
else:
name = obj.name
if url: if url:
el = self.extension.make_link(category, obj, url, label or name) el = self.factory.make_link(category, obj, url, label or name)
else: else:
el = markdown.etree.Element('span') el = markdown.etree.Element('span')
el.text = markdown.AtomicString(label or name) el.text = markdown.AtomicString(label or name)
return el return el
class PokedexLinkExtension(markdown.Extension): class MarkdownLinkMaker(object):
"""Plugs the [foo]{bar:baz} syntax into the markdown parser. """Creates Markdown extensions for handling links for the given session.
Subclases need to set the `session` attribute to the current session, There are two ways to customize the link handling: either override the
and `language` to the language of the strings. *_url methods in a subclass, or give them as arguments to get_extension
(or MarkdownString.as_html).
To get links, subclasses must override object_url and/or identifier_url.
If these return None, <span>s are used instead of <a>.
""" """
language = None def __init__(self, session=None):
self.session = session
def __init__(self):
markdown.Extension.__init__(self)
self.link_pattern = PokedexLinkPattern(self)
def get_extension(self, language=None, object_url=None, identifier_url=None,
make_link=None):
"""Get a Markdown extension that handles links using the given language.
"""
link_maker = self
class LinkExtension(markdown.Extension):
def extendMarkdown(self, md, md_globals): def extendMarkdown(self, md, md_globals):
md.inlinePatterns['pokedex-link'] = self.link_pattern self.identifier_url = identifier_url or link_maker.identifier_url
self.object_url = object_url or link_maker.object_url
self.make_link = make_link or link_maker.make_link
self.session = link_maker.session
pattern = PokedexLinkPattern(self, language)
md.inlinePatterns['pokedex-link'] = pattern
return LinkExtension()
def make_link(self, category, obj, url, text): def make_link(self, category, obj, url, text):
"""Make an <a> element """Make an <a> element
@ -216,8 +221,7 @@ class PokedexLinkExtension(markdown.Extension):
def identifier_url(self, category, identifier): def identifier_url(self, category, identifier):
"""Return the URL for the given {category:identifier} link """Return the URL for the given {category:identifier} link
For ORM objects, object_url is used instead (but may fall back to For ORM objects, object_url is tried first
identifier_url).
Returns None by default, which causes <span> to be used in place of <a> Returns None by default, which causes <span> to be used in place of <a>
""" """
@ -226,16 +230,6 @@ class PokedexLinkExtension(markdown.Extension):
def object_url(self, category, obj): def object_url(self, category, obj):
"""Return the URL for the ORM object obj """Return the URL for the ORM object obj
Calls identifier_url by default. Returns None by default, which causes identifier_url to be used
""" """
return self.identifier_url(category, obj.identifier) return None
class ParametrizedLinkExtension(PokedexLinkExtension):
def __init__(self, session, object_url=None, identifier_url=None, language=None):
PokedexLinkExtension.__init__(self)
self.language = language
self.session = session
if object_url:
self.object_url = object_url
if identifier_url:
self.identifier_url = identifier_url

View file

@ -9,6 +9,8 @@ from sqlalchemy.schema import Column, ForeignKey, Table
from sqlalchemy.sql.expression import and_, bindparam, select from sqlalchemy.sql.expression import and_, bindparam, select
from sqlalchemy.types import Integer from sqlalchemy.types import Integer
from pokedex.db import markdown
def create_translation_table(_table_name, foreign_class, relation_name, def create_translation_table(_table_name, foreign_class, relation_name,
language_class, relation_lazy='select', **kwargs): language_class, relation_lazy='select', **kwargs):
"""Creates a table that represents some kind of data attached to the given """Creates a table that represents some kind of data attached to the given
@ -64,6 +66,9 @@ def create_translation_table(_table_name, foreign_class, relation_name,
Pardon the naming disparity, but the grammar suffers otherwise. Pardon the naming disparity, but the grammar suffers otherwise.
Modifying these directly is not likely to be a good idea. Modifying these directly is not likely to be a good idea.
For Markdown-formatted columns, `(column)_map` and `(column)` will give
Markdown objects.
""" """
# n.b.: language_class only exists for the sake of tests, which sometimes # n.b.: language_class only exists for the sake of tests, which sometimes
# want to create tables entirely separate from the pokedex metadata # want to create tables entirely separate from the pokedex metadata
@ -132,9 +137,26 @@ def create_translation_table(_table_name, foreign_class, relation_name,
# Add per-column proxies to the original class # Add per-column proxies to the original class
for name, column in kwitems: for name, column in kwitems:
string_getter = column.info.get('string_getter')
if string_getter:
def getset_factory(underlying_type, instance):
def getter(translations):
text = getattr(translations, column.name)
session = object_session(translations)
language = translations.local_language
return string_getter(text, session, language)
def setter(translations, value):
# The string must be set on the Translation directly.
raise AttributeError("Cannot set %s" % column.name)
return getter, setter
getset_factory = getset_factory
else:
getset_factory = None
# Class.(column) -- accessor for the default language's value # Class.(column) -- accessor for the default language's value
setattr(foreign_class, name, setattr(foreign_class, name,
association_proxy(local_relation_name, name)) association_proxy(local_relation_name, name,
getset_factory=getset_factory))
# Class.(column)_map -- accessor for the language dict # Class.(column)_map -- accessor for the language dict
# Need a custom creator since Translations doesn't have an init, and # Need a custom creator since Translations doesn't have an init, and
@ -145,7 +167,8 @@ def create_translation_table(_table_name, foreign_class, relation_name,
setattr(row, name, value) setattr(row, name, value)
return row return row
setattr(foreign_class, name + '_map', setattr(foreign_class, name + '_map',
association_proxy(relation_name, name, creator=creator)) association_proxy(relation_name, name, creator=creator,
getset_factory=getset_factory))
# Add to the list of translation classes # Add to the list of translation classes
foreign_class.translation_classes.append(Translations) foreign_class.translation_classes.append(Translations)
@ -164,6 +187,8 @@ class MultilangSession(Session):
if 'default_language_id' in kwargs: if 'default_language_id' in kwargs:
self.default_language_id = kwargs.pop('default_language_id') self.default_language_id = kwargs.pop('default_language_id')
self.pokedex_link_maker = markdown.MarkdownLinkMaker(self)
super(MultilangSession, self).__init__(*args, **kwargs) super(MultilangSession, self).__init__(*args, **kwargs)
def execute(self, clause, params=None, *args, **kwargs): def execute(self, clause, params=None, *args, **kwargs):
@ -189,3 +214,13 @@ class MultilangScopedSession(ScopedSession):
@default_language_id.setter @default_language_id.setter
def default_language_id(self, new): def default_language_id(self, new):
self.registry().default_language_id = new self.registry().default_language_id = new
@property
def pokedex_link_maker(self):
"""Passes the new link maker through to the current session.
"""
return self.registry().pokedex_link_maker
@pokedex_link_maker.setter
def pokedex_link_maker(self, new):
self.registry().pokedex_link_maker = new

View file

@ -18,6 +18,10 @@ Columns have a info dictionary with these keys:
- ripped: True for text that has been ripped from the games, and can be ripped - ripped: True for text that has been ripped from the games, and can be ripped
again for new versions or languages again for new versions or languages
- string_getter: for translation columns, a function taking (text, session,
language) that is used for properties on the main table. Used for Markdown
text.
See `pokedex.db.multilang` for how localizable text columns work. The session See `pokedex.db.multilang` for how localizable text columns work. The session
classes in that module can be used to change the default language. classes in that module can be used to change the default language.
""" """
@ -124,10 +128,10 @@ create_translation_table('ability_names', Ability, 'names',
info=dict(description="The name", format='plaintext', official=True, ripped=True)), info=dict(description="The name", format='plaintext', official=True, ripped=True)),
) )
create_translation_table('ability_prose', Ability, 'prose', create_translation_table('ability_prose', Ability, 'prose',
effect = Column(markdown.MarkdownColumn(5120), nullable=True, effect = Column(Unicode(5120), nullable=True,
info=dict(description="A detailed description of this ability's effect", format='markdown')), info=dict(description="A detailed description of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)),
short_effect = Column(markdown.MarkdownColumn(512), nullable=True, short_effect = Column(Unicode(512), nullable=True,
info=dict(description="A short summary of this ability's effect", format='markdown')), info=dict(description="A short summary of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)),
) )
class AbilityChangelog(TableBase): class AbilityChangelog(TableBase):
@ -142,8 +146,8 @@ class AbilityChangelog(TableBase):
info=dict(description="The ID of the version group in which the ability changed")) info=dict(description="The ID of the version group in which the ability changed"))
create_translation_table('ability_changelog_prose', AbilityChangelog, 'prose', create_translation_table('ability_changelog_prose', AbilityChangelog, 'prose',
effect = Column(markdown.MarkdownColumn(255), nullable=False, effect = Column(Unicode(255), nullable=False,
info=dict(description="A description of the old behavior", format='markdown')) info=dict(description="A description of the old behavior", format='markdown', string_getter=markdown.MarkdownString))
) )
class AbilityFlavorText(TableBase): class AbilityFlavorText(TableBase):
@ -503,10 +507,10 @@ create_translation_table('item_names', Item, 'names',
info=dict(description="The name", format='plaintext', official=True, ripped=True)), info=dict(description="The name", format='plaintext', official=True, ripped=True)),
) )
create_translation_table('item_prose', Item, 'prose', create_translation_table('item_prose', Item, 'prose',
short_effect = Column(markdown.MarkdownColumn(256), nullable=True, short_effect = Column(Unicode(256), nullable=True,
info=dict(description="A short summary of the effect", format='markdown')), info=dict(description="A short summary of the effect", format='markdown', string_getter=markdown.MarkdownString)),
effect = Column(markdown.MarkdownColumn(5120), nullable=True, effect = Column(Unicode(5120), nullable=True,
info=dict(description=u"Detailed description of the item's effect.", format='markdown')), info=dict(description=u"Detailed description of the item's effect.", format='markdown', string_getter=markdown.MarkdownString)),
) )
create_translation_table('item_flavor_summaries', Item, 'flavor_summaries', create_translation_table('item_flavor_summaries', Item, 'flavor_summaries',
flavor_summary = Column(Unicode(512), nullable=True, flavor_summary = Column(Unicode(512), nullable=True,
@ -803,7 +807,7 @@ class MoveEffect(TableBase):
create_translation_table('move_effect_prose', MoveEffect, 'prose', create_translation_table('move_effect_prose', MoveEffect, 'prose',
short_effect = Column(Unicode(256), nullable=True, short_effect = Column(Unicode(256), nullable=True,
info=dict(description="A short summary of the effect", format='plaintext')), info=dict(description="A short summary of the effect", format='markdown')),
effect = Column(Unicode(5120), nullable=True, effect = Column(Unicode(5120), nullable=True,
info=dict(description="A detailed description of the effect", format='markdown')), info=dict(description="A detailed description of the effect", format='markdown')),
) )
@ -825,8 +829,8 @@ class MoveEffectChangelog(TableBase):
) )
create_translation_table('move_effect_changelog_prose', MoveEffectChangelog, 'prose', create_translation_table('move_effect_changelog_prose', MoveEffectChangelog, 'prose',
effect = Column(markdown.MarkdownColumn(512), nullable=False, effect = Column(Unicode(512), nullable=False,
info=dict(description="A description of the old behavior", format='markdown')), info=dict(description="A description of the old behavior", format='markdown', string_getter=markdown.MarkdownString)),
) )
class MoveFlag(TableBase): class MoveFlag(TableBase):
@ -854,8 +858,8 @@ create_translation_table('move_flag_type_prose', MoveFlagType, 'prose',
relation_lazy='joined', relation_lazy='joined',
name = Column(Unicode(32), nullable=True, index=True, name = Column(Unicode(32), nullable=True, index=True,
info=dict(description="The name", format='plaintext', official=False)), info=dict(description="The name", format='plaintext', official=False)),
description = Column(markdown.MarkdownColumn(256), nullable=True, description = Column(Unicode(256), nullable=True,
info=dict(description="A short description of the flag", format='markdown')), info=dict(description="A short description of the flag", format='markdown', string_getter=markdown.MarkdownString)),
) )
class MoveFlavorText(TableBase): class MoveFlavorText(TableBase):
@ -1355,8 +1359,8 @@ PokemonFormGroup.id = PokemonFormGroup.pokemon_id
create_translation_table('pokemon_form_group_prose', PokemonFormGroup, 'prose', create_translation_table('pokemon_form_group_prose', PokemonFormGroup, 'prose',
term = Column(Unicode(16), nullable=True, term = Column(Unicode(16), nullable=True,
info=dict(description=u"The term for this Pokémon's forms, e.g. \"Cloak\" for Burmy or \"Forme\" for Deoxys.", official=True, format='plaintext')), info=dict(description=u"The term for this Pokémon's forms, e.g. \"Cloak\" for Burmy or \"Forme\" for Deoxys.", official=True, format='plaintext')),
description = Column(markdown.MarkdownColumn(1024), nullable=True, description = Column(Unicode(1024), nullable=True,
info=dict(description=u"Description of how the forms work", format='markdown')), info=dict(description=u"Description of how the forms work", format='markdown', string_getter=markdown.MarkdownString)),
) )
class PokemonFormPokeathlonStat(TableBase): class PokemonFormPokeathlonStat(TableBase):

View file

@ -180,17 +180,13 @@ def test_texts():
if format is not None: if format is not None:
if format not in good_formats: if format not in good_formats:
raise AssertionError(assert_text % column) raise AssertionError(assert_text % column)
is_markdown = isinstance(column.type, markdown.MarkdownColumn)
if is_markdown and (format != 'markdown'):
# Note: regular string with markdown syntax is allowed
raise AssertionError('%s: markdown format/column type mismatch' % column)
if (format != 'identifier') and (column.name == 'identifier'): if (format != 'identifier') and (column.name == 'identifier'):
raise AssertionError('%s: identifier column name/type mismatch' % column) raise AssertionError('%s: identifier column name/type mismatch' % column)
if column.info.get('official', None) and format not in 'gametext plaintext': if column.info.get('official', None) and format not in 'gametext plaintext':
raise AssertionError('%s: official text with bad format' % column) raise AssertionError('%s: official text with bad format' % column)
text_columns.append(column) text_columns.append(column)
else: else:
if isinstance(column.type, (markdown.MarkdownColumn, tables.Unicode)): if isinstance(column.type, tables.Unicode):
raise AssertionError('%s: text column without format' % column) raise AssertionError('%s: text column without format' % column)
if column.name == 'name' and format != 'plaintext': if column.name == 'name' and format != 'plaintext':
raise AssertionError('%s: non-plaintext name' % column) raise AssertionError('%s: non-plaintext name' % column)

View file

@ -2,7 +2,7 @@
from nose.tools import * from nose.tools import *
from pokedex.db import tables, connect from pokedex.db import tables, connect, util, markdown
class TestStrings(object): class TestStrings(object):
def setup(self): def setup(self):
@ -83,5 +83,22 @@ class TestStrings(object):
identifier=u"thunderbolt").one() identifier=u"thunderbolt").one()
language = self.connection.query(tables.Language).filter_by( language = self.connection.query(tables.Language).filter_by(
identifier=u"en").one() identifier=u"en").one()
assert '10%' in move.effect.as_text assert '10%' in move.effect.as_text()
assert '10%' in move.effect_map[language].as_text assert '10%' in move.effect_map[language].as_text()
assert '10%' in move.effect.as_html()
assert '10%' in move.effect_map[language].as_html()
assert '10%' in unicode(move.effect)
assert '10%' in unicode(move.effect_map[language])
assert '10%' in move.effect.__html__()
assert '10%' in move.effect_map[language].__html__()
def test_markdown_string(self):
en = util.get(self.connection, tables.Language, 'en')
md = markdown.MarkdownString('[]{move:thunderbolt} [paralyzes]{mechanic:paralysis}', self.connection, en)
assert unicode(md) == 'Thunderbolt paralyzes'
assert md.as_html() == '<p><span>Thunderbolt</span> <span>paralyzes</span></p>'
assert md.as_html(object_url=lambda category, obj: "%s/%s" % (category, obj.identifier)) == (
'<p><a href="move/thunderbolt">Thunderbolt</a> <span>paralyzes</span></p>')
print md.as_html(identifier_url=lambda category, ident: "%s/%s" % (category, ident))
assert md.as_html(identifier_url=lambda category, ident: "%s/%s" % (category, ident)) == (
'<p><a href="move/thunderbolt">Thunderbolt</a> <a href="mechanic/paralysis">paralyzes</a></p>')