diff --git a/pokedex/db/markdown.py b/pokedex/db/markdown.py index 81e6e65..9653a53 100644 --- a/pokedex/db/markdown.py +++ b/pokedex/db/markdown.py @@ -5,9 +5,9 @@ The language used is a variation of Markdown and Markdown Extra. There are docs for each at http://daringfireball.net/projects/markdown/ and http://michelf.com/projects/php-markdown/extra/ respectively. -Pokédex links are represented with the syntax `[text]{type:identifier}`, e.g., -`[Eevee]{pokemon:eevee}`. The actual code that parses these is in -spline-pokedex. +Pokédex links are represented with the syntax `[label]{category:identifier}`, +e.g., `[Eevee]{pokemon:eevee}`. The label can (and should) be left out, in +which case it is replaced by the name of the thing linked to. """ from __future__ import absolute_import @@ -15,40 +15,52 @@ import re import markdown import sqlalchemy.types +from sqlalchemy.orm.session import object_session class MarkdownString(object): - """Wraps a Markdown string. Stringifies to the original text, but .as_html - will return an HTML rendering. + """Wraps a Markdown string. - To make the __html__ property work, you must set this class's - `default_link_extension` to a PokedexLinkExtension. Yep, that's gross. + Use unicode() and __html__ for text and HTML representations. + The as_text() and as_html() functions do the same, but accept optional + arguments that may affect the rendering. + The `source_text` property holds the original text. + + init args: + `source_text`: the text in Markdown syntax + `session`: A DB session used for looking up linked objects + `language`: The language the string is in. If None, the session default + is used. """ default_link_extension = None - def __init__(self, source_text): + def __init__(self, source_text, session, language): self.source_text = source_text + self.session = session + self.language = language def __unicode__(self): - return self.source_text + return self.as_text() def __str__(self): - return unicode(self.source_text).encode() + return self.as_text().encode() def __html__(self): - return self.as_html(extension=self.default_link_extension) + return self.as_html() - def as_html(self, session=None, object_url=None, identifier_url=None, language=None, extension=None): + def as_html(self, object_url=None, identifier_url=None, make_link=None): """Returns the string as HTML. - Pass in current session, and optionally URL-making functions and the - language. See PokedexLinkExtension for how they work. - - Alternatively, pass in a PokedexLinkExtension instance. + If given, the optional arguments will be used instead of those in the + session's pokedex_link_maker. See MarkdownLinkMaker for documentation. """ - if not extension: - extension = ParametrizedLinkExtension(session, object_url, identifier_url, language) + extension = self.session.pokedex_link_maker.get_extension( + self.language, + object_url=object_url, + identifier_url=identifier_url, + make_link=make_link, + ) md = markdown.Markdown( extensions=['extra', extension], @@ -58,20 +70,26 @@ class MarkdownString(object): return md.convert(self.source_text) - def as_text(self, session): + def as_text(self): """Returns the string in a plaintext-friendly form. + + Currently there are no tunable parameters """ # Since Markdown is pretty readable by itself, we just have to replace # the links by their text. # XXX: The tables get unaligned - extension = ParametrizedLinkExtension(session) - pattern = extension.link_pattern + + link_maker = MarkdownLinkMaker(self.session) + pattern = PokedexLinkPattern(link_maker, self.language) regex = '()%s()' % pattern.regex def handleMatch(m): return pattern.handleMatch(m).text + return re.sub(regex, handleMatch, self.source_text) -def _markdownify_effect_text(move, effect_text): +def _markdownify_effect_text(move, effect_text, language=None): + session = object_session(move) + if effect_text is None: return effect_text effect_text = effect_text.replace( @@ -79,7 +97,7 @@ def _markdownify_effect_text(move, effect_text): unicode(move.effect_chance), ) - return MarkdownString(effect_text) + return MarkdownString(effect_text, session, language) class MoveEffectProperty(object): """Property that wraps move effects. Used like this: @@ -110,42 +128,22 @@ class MoveEffectPropertyMap(MoveEffectProperty): prop = getattr(obj.move_effect, self.effect_column) newdict = dict(prop) for key in newdict: - newdict[key] = _markdownify_effect_text(obj, newdict[key]) + newdict[key] = _markdownify_effect_text(obj, newdict[key], key) return newdict -class MarkdownColumn(sqlalchemy.types.TypeDecorator): - """Generic SQLAlchemy column type for Markdown text. - - Do NOT use this for move effects! They need to know what move they belong - to so they can fill in, e.g., effect chances. Use the MoveEffectProperty - property class above. - """ - impl = sqlalchemy.types.Unicode - - def process_bind_param(self, value, dialect): - if value is None: - return None - - if not isinstance(value, basestring): - # Can't assign, e.g., MarkdownString objects yet - raise NotImplementedError - - return unicode(value) - - def process_result_value(self, value, dialect): - if value is None: - return None - - return MarkdownString(value) - class PokedexLinkPattern(markdown.inlinepatterns.Pattern): """Matches [label]{category:target}. + + Handles matches using factory """ regex = ur'(?x) \[ ([^]]*) \] \{ ([-a-z0-9]+) : ([-a-z0-9]+) \}' - def __init__(self, extension): + def __init__(self, factory, string_language, game_language=None): markdown.inlinepatterns.Pattern.__init__(self, self.regex) - self.extension = extension + self.factory = factory + self.session = factory.session + self.string_language = string_language + self.game_language = game_language def handleMatch(self, m): from pokedex.db import tables, util @@ -161,47 +159,54 @@ class PokedexLinkPattern(markdown.inlinepatterns.Pattern): )[category] except KeyError: obj = name = target - url = self.extension.identifier_url(category, obj) + url = self.factory.identifier_url(category, obj) else: - session = self.extension.session - obj = util.get(self.extension.session, table, target) - url = self.extension.object_url(category, obj) + session = self.session + obj = util.get(self.session, table, target) + url = self.factory.object_url(category, obj) + url = url or self.factory.identifier_url(category, obj.identifier) + name = None + # Translations can be incomplete; in which case we want to use a + # fallback. if table in [tables.Type]: # Type wants to be localized to the same language as the text - language = self.extension.language - name = None - try: - name = obj.name_map[language] - except KeyError: - pass - if not name: - name = obj.name - else: + name = obj.name_map.get(self.string_language) + if not name and self.game_language: + name = obj.name_map.get(self.game_language) + if not name: name = obj.name if url: - el = self.extension.make_link(category, obj, url, label or name) + el = self.factory.make_link(category, obj, url, label or name) else: el = markdown.etree.Element('span') el.text = markdown.AtomicString(label or name) return el -class PokedexLinkExtension(markdown.Extension): - """Plugs the [foo]{bar:baz} syntax into the markdown parser. +class MarkdownLinkMaker(object): + """Creates Markdown extensions for handling links for the given session. - Subclases need to set the `session` attribute to the current session, - and `language` to the language of the strings. - - To get links, subclasses must override object_url and/or identifier_url. - If these return None, s are used instead of . + There are two ways to customize the link handling: either override the + *_url methods in a subclass, or give them as arguments to get_extension + (or MarkdownString.as_html). """ - language = None + def __init__(self, session=None): + self.session = session - def __init__(self): - markdown.Extension.__init__(self) - self.link_pattern = PokedexLinkPattern(self) + def get_extension(self, language=None, object_url=None, identifier_url=None, + make_link=None): + """Get a Markdown extension that handles links using the given language. + """ + link_maker = self + class LinkExtension(markdown.Extension): + def extendMarkdown(self, md, md_globals): + self.identifier_url = identifier_url or link_maker.identifier_url + self.object_url = object_url or link_maker.object_url + self.make_link = make_link or link_maker.make_link + self.session = link_maker.session + pattern = PokedexLinkPattern(self, language) + md.inlinePatterns['pokedex-link'] = pattern - def extendMarkdown(self, md, md_globals): - md.inlinePatterns['pokedex-link'] = self.link_pattern + return LinkExtension() def make_link(self, category, obj, url, text): """Make an element @@ -216,8 +221,7 @@ class PokedexLinkExtension(markdown.Extension): def identifier_url(self, category, identifier): """Return the URL for the given {category:identifier} link - For ORM objects, object_url is used instead (but may fall back to - identifier_url). + For ORM objects, object_url is tried first Returns None by default, which causes to be used in place of """ @@ -226,16 +230,6 @@ class PokedexLinkExtension(markdown.Extension): def object_url(self, category, obj): """Return the URL for the ORM object obj - Calls identifier_url by default. + Returns None by default, which causes identifier_url to be used """ - return self.identifier_url(category, obj.identifier) - -class ParametrizedLinkExtension(PokedexLinkExtension): - def __init__(self, session, object_url=None, identifier_url=None, language=None): - PokedexLinkExtension.__init__(self) - self.language = language - self.session = session - if object_url: - self.object_url = object_url - if identifier_url: - self.identifier_url = identifier_url + return None diff --git a/pokedex/db/multilang.py b/pokedex/db/multilang.py index f66ee20..0c9a188 100644 --- a/pokedex/db/multilang.py +++ b/pokedex/db/multilang.py @@ -9,6 +9,8 @@ from sqlalchemy.schema import Column, ForeignKey, Table from sqlalchemy.sql.expression import and_, bindparam, select from sqlalchemy.types import Integer +from pokedex.db import markdown + def create_translation_table(_table_name, foreign_class, relation_name, language_class, relation_lazy='select', **kwargs): """Creates a table that represents some kind of data attached to the given @@ -64,6 +66,9 @@ def create_translation_table(_table_name, foreign_class, relation_name, Pardon the naming disparity, but the grammar suffers otherwise. Modifying these directly is not likely to be a good idea. + + For Markdown-formatted columns, `(column)_map` and `(column)` will give + Markdown objects. """ # n.b.: language_class only exists for the sake of tests, which sometimes # want to create tables entirely separate from the pokedex metadata @@ -132,9 +137,26 @@ def create_translation_table(_table_name, foreign_class, relation_name, # Add per-column proxies to the original class for name, column in kwitems: + string_getter = column.info.get('string_getter') + if string_getter: + def getset_factory(underlying_type, instance): + def getter(translations): + text = getattr(translations, column.name) + session = object_session(translations) + language = translations.local_language + return string_getter(text, session, language) + def setter(translations, value): + # The string must be set on the Translation directly. + raise AttributeError("Cannot set %s" % column.name) + return getter, setter + getset_factory = getset_factory + else: + getset_factory = None + # Class.(column) -- accessor for the default language's value setattr(foreign_class, name, - association_proxy(local_relation_name, name)) + association_proxy(local_relation_name, name, + getset_factory=getset_factory)) # Class.(column)_map -- accessor for the language dict # Need a custom creator since Translations doesn't have an init, and @@ -145,7 +167,8 @@ def create_translation_table(_table_name, foreign_class, relation_name, setattr(row, name, value) return row setattr(foreign_class, name + '_map', - association_proxy(relation_name, name, creator=creator)) + association_proxy(relation_name, name, creator=creator, + getset_factory=getset_factory)) # Add to the list of translation classes foreign_class.translation_classes.append(Translations) @@ -164,6 +187,8 @@ class MultilangSession(Session): if 'default_language_id' in kwargs: self.default_language_id = kwargs.pop('default_language_id') + self.pokedex_link_maker = markdown.MarkdownLinkMaker(self) + super(MultilangSession, self).__init__(*args, **kwargs) def execute(self, clause, params=None, *args, **kwargs): @@ -189,3 +214,13 @@ class MultilangScopedSession(ScopedSession): @default_language_id.setter def default_language_id(self, new): self.registry().default_language_id = new + + @property + def pokedex_link_maker(self): + """Passes the new link maker through to the current session. + """ + return self.registry().pokedex_link_maker + + @pokedex_link_maker.setter + def pokedex_link_maker(self, new): + self.registry().pokedex_link_maker = new diff --git a/pokedex/db/tables.py b/pokedex/db/tables.py index 411c712..bc78ff6 100644 --- a/pokedex/db/tables.py +++ b/pokedex/db/tables.py @@ -18,6 +18,10 @@ Columns have a info dictionary with these keys: - ripped: True for text that has been ripped from the games, and can be ripped again for new versions or languages + - string_getter: for translation columns, a function taking (text, session, + language) that is used for properties on the main table. Used for Markdown + text. + See `pokedex.db.multilang` for how localizable text columns work. The session classes in that module can be used to change the default language. """ @@ -124,10 +128,10 @@ create_translation_table('ability_names', Ability, 'names', info=dict(description="The name", format='plaintext', official=True, ripped=True)), ) create_translation_table('ability_prose', Ability, 'prose', - effect = Column(markdown.MarkdownColumn(5120), nullable=True, - info=dict(description="A detailed description of this ability's effect", format='markdown')), - short_effect = Column(markdown.MarkdownColumn(512), nullable=True, - info=dict(description="A short summary of this ability's effect", format='markdown')), + effect = Column(Unicode(5120), nullable=True, + info=dict(description="A detailed description of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)), + short_effect = Column(Unicode(512), nullable=True, + info=dict(description="A short summary of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)), ) class AbilityChangelog(TableBase): @@ -142,8 +146,8 @@ class AbilityChangelog(TableBase): info=dict(description="The ID of the version group in which the ability changed")) create_translation_table('ability_changelog_prose', AbilityChangelog, 'prose', - effect = Column(markdown.MarkdownColumn(255), nullable=False, - info=dict(description="A description of the old behavior", format='markdown')) + effect = Column(Unicode(255), nullable=False, + info=dict(description="A description of the old behavior", format='markdown', string_getter=markdown.MarkdownString)) ) class AbilityFlavorText(TableBase): @@ -503,10 +507,10 @@ create_translation_table('item_names', Item, 'names', info=dict(description="The name", format='plaintext', official=True, ripped=True)), ) create_translation_table('item_prose', Item, 'prose', - short_effect = Column(markdown.MarkdownColumn(256), nullable=True, - info=dict(description="A short summary of the effect", format='markdown')), - effect = Column(markdown.MarkdownColumn(5120), nullable=True, - info=dict(description=u"Detailed description of the item's effect.", format='markdown')), + short_effect = Column(Unicode(256), nullable=True, + info=dict(description="A short summary of the effect", format='markdown', string_getter=markdown.MarkdownString)), + effect = Column(Unicode(5120), nullable=True, + info=dict(description=u"Detailed description of the item's effect.", format='markdown', string_getter=markdown.MarkdownString)), ) create_translation_table('item_flavor_summaries', Item, 'flavor_summaries', flavor_summary = Column(Unicode(512), nullable=True, @@ -803,7 +807,7 @@ class MoveEffect(TableBase): create_translation_table('move_effect_prose', MoveEffect, 'prose', short_effect = Column(Unicode(256), nullable=True, - info=dict(description="A short summary of the effect", format='plaintext')), + info=dict(description="A short summary of the effect", format='markdown')), effect = Column(Unicode(5120), nullable=True, info=dict(description="A detailed description of the effect", format='markdown')), ) @@ -825,8 +829,8 @@ class MoveEffectChangelog(TableBase): ) create_translation_table('move_effect_changelog_prose', MoveEffectChangelog, 'prose', - effect = Column(markdown.MarkdownColumn(512), nullable=False, - info=dict(description="A description of the old behavior", format='markdown')), + effect = Column(Unicode(512), nullable=False, + info=dict(description="A description of the old behavior", format='markdown', string_getter=markdown.MarkdownString)), ) class MoveFlag(TableBase): @@ -854,8 +858,8 @@ create_translation_table('move_flag_type_prose', MoveFlagType, 'prose', relation_lazy='joined', name = Column(Unicode(32), nullable=True, index=True, info=dict(description="The name", format='plaintext', official=False)), - description = Column(markdown.MarkdownColumn(256), nullable=True, - info=dict(description="A short description of the flag", format='markdown')), + description = Column(Unicode(256), nullable=True, + info=dict(description="A short description of the flag", format='markdown', string_getter=markdown.MarkdownString)), ) class MoveFlavorText(TableBase): @@ -1355,8 +1359,8 @@ PokemonFormGroup.id = PokemonFormGroup.pokemon_id create_translation_table('pokemon_form_group_prose', PokemonFormGroup, 'prose', term = Column(Unicode(16), nullable=True, info=dict(description=u"The term for this Pokémon's forms, e.g. \"Cloak\" for Burmy or \"Forme\" for Deoxys.", official=True, format='plaintext')), - description = Column(markdown.MarkdownColumn(1024), nullable=True, - info=dict(description=u"Description of how the forms work", format='markdown')), + description = Column(Unicode(1024), nullable=True, + info=dict(description=u"Description of how the forms work", format='markdown', string_getter=markdown.MarkdownString)), ) class PokemonFormPokeathlonStat(TableBase): diff --git a/pokedex/tests/test_schema.py b/pokedex/tests/test_schema.py index eb536ec..c09564a 100644 --- a/pokedex/tests/test_schema.py +++ b/pokedex/tests/test_schema.py @@ -180,17 +180,13 @@ def test_texts(): if format is not None: if format not in good_formats: raise AssertionError(assert_text % column) - is_markdown = isinstance(column.type, markdown.MarkdownColumn) - if is_markdown and (format != 'markdown'): - # Note: regular string with markdown syntax is allowed - raise AssertionError('%s: markdown format/column type mismatch' % column) if (format != 'identifier') and (column.name == 'identifier'): raise AssertionError('%s: identifier column name/type mismatch' % column) if column.info.get('official', None) and format not in 'gametext plaintext': raise AssertionError('%s: official text with bad format' % column) text_columns.append(column) else: - if isinstance(column.type, (markdown.MarkdownColumn, tables.Unicode)): + if isinstance(column.type, tables.Unicode): raise AssertionError('%s: text column without format' % column) if column.name == 'name' and format != 'plaintext': raise AssertionError('%s: non-plaintext name' % column) diff --git a/pokedex/tests/test_strings.py b/pokedex/tests/test_strings.py index 8d0fa06..dd02e7d 100644 --- a/pokedex/tests/test_strings.py +++ b/pokedex/tests/test_strings.py @@ -2,7 +2,7 @@ from nose.tools import * -from pokedex.db import tables, connect +from pokedex.db import tables, connect, util, markdown class TestStrings(object): def setup(self): @@ -83,5 +83,22 @@ class TestStrings(object): identifier=u"thunderbolt").one() language = self.connection.query(tables.Language).filter_by( identifier=u"en").one() - assert '10%' in move.effect.as_text - assert '10%' in move.effect_map[language].as_text + assert '10%' in move.effect.as_text() + assert '10%' in move.effect_map[language].as_text() + assert '10%' in move.effect.as_html() + assert '10%' in move.effect_map[language].as_html() + assert '10%' in unicode(move.effect) + assert '10%' in unicode(move.effect_map[language]) + assert '10%' in move.effect.__html__() + assert '10%' in move.effect_map[language].__html__() + + def test_markdown_string(self): + en = util.get(self.connection, tables.Language, 'en') + md = markdown.MarkdownString('[]{move:thunderbolt} [paralyzes]{mechanic:paralysis}', self.connection, en) + assert unicode(md) == 'Thunderbolt paralyzes' + assert md.as_html() == '

Thunderbolt paralyzes

' + assert md.as_html(object_url=lambda category, obj: "%s/%s" % (category, obj.identifier)) == ( + '

Thunderbolt paralyzes

') + print md.as_html(identifier_url=lambda category, ident: "%s/%s" % (category, ident)) + assert md.as_html(identifier_url=lambda category, ident: "%s/%s" % (category, ident)) == ( + '

Thunderbolt paralyzes

')