Move Markdown handling to the translation classes

- the Session has a `pokedex_link_maker` property, whose `object_url`
  method is used to make URLs in Markdown
- pokemon.names_table.name is now an ordinary Unicode column
- pokemon.name is a MarkdownString that is aware of the session and the
  language the string is in
- pokemon.name_map is a dict-like association_proxy of the above
- move.effect works similarly, with transparent $effect_chance substitution
  as before

pokedex/db/markdown.py

@@ -5,9 +5,9 @@ The language used is a variation of Markdown and Markdown Extra.  There are
 docs for each at http://daringfireball.net/projects/markdown/ and
 http://michelf.com/projects/php-markdown/extra/ respectively.
 
-Pokédex links are represented with the syntax `[text]{type:identifier}`, e.g.,
-`[Eevee]{pokemon:eevee}`.  The actual code that parses these is in
-spline-pokedex.
+Pokédex links are represented with the syntax `[label]{category:identifier}`,
+e.g., `[Eevee]{pokemon:eevee}`. The label can (and should) be left out, in
+which case it is replaced by the name of the thing linked to.
 """
 from __future__ import absolute_import
 
@@ -15,40 +15,52 @@ import re
 
 import markdown
 import sqlalchemy.types
+from sqlalchemy.orm.session import object_session
 
 class MarkdownString(object):
-    """Wraps a Markdown string.  Stringifies to the original text, but .as_html
-    will return an HTML rendering.
+    """Wraps a Markdown string.
 
-    To make the __html__ property work, you must set this class's
-    `default_link_extension` to a PokedexLinkExtension.  Yep, that's gross.
+    Use unicode() and __html__ for text and HTML representations.
+    The as_text() and as_html() functions do the same, but accept optional
+    arguments that may affect the rendering.
+    The `source_text` property holds the original text.
+
+    init args:
+    `source_text`: the text in Markdown syntax
+    `session`: A DB session used for looking up linked objects
+    `language`: The language the string is in. If None, the session default
+        is used.
     """
 
     default_link_extension = None
 
-    def __init__(self, source_text):
+    def __init__(self, source_text, session, language):
         self.source_text = source_text
+        self.session = session
+        self.language = language
 
     def __unicode__(self):
-        return self.source_text
+        return self.as_text()
 
     def __str__(self):
-        return unicode(self.source_text).encode()
+        return self.as_text().encode()
 
     def __html__(self):
-        return self.as_html(extension=self.default_link_extension)
+        return self.as_html()
 
-    def as_html(self, session=None, object_url=None, identifier_url=None, language=None, extension=None):
+    def as_html(self, object_url=None, identifier_url=None, make_link=None):
         """Returns the string as HTML.
 
-        Pass in current session, and optionally URL-making functions and the
-        language. See PokedexLinkExtension for how they work.
-
-        Alternatively, pass in a PokedexLinkExtension instance.
+        If given, the optional arguments will be used instead of those in the
+        session's pokedex_link_maker. See MarkdownLinkMaker for documentation.
         """
 
-        if not extension:
-            extension = ParametrizedLinkExtension(session, object_url, identifier_url, language)
+        extension = self.session.pokedex_link_maker.get_extension(
+                self.language,
+                object_url=object_url,
+                identifier_url=identifier_url,
+                make_link=make_link,
+            )
 
         md = markdown.Markdown(
             extensions=['extra', extension],
@@ -58,20 +70,26 @@ class MarkdownString(object):
 
         return md.convert(self.source_text)
 
-    def as_text(self, session):
+    def as_text(self):
         """Returns the string in a plaintext-friendly form.
+
+        Currently there are no tunable parameters
         """
         # Since Markdown is pretty readable by itself, we just have to replace
         # the links by their text.
         # XXX: The tables get unaligned
-        extension = ParametrizedLinkExtension(session)
-        pattern = extension.link_pattern
+
+        link_maker = MarkdownLinkMaker(self.session)
+        pattern = PokedexLinkPattern(link_maker, self.language)
         regex = '()%s()' % pattern.regex
         def handleMatch(m):
             return pattern.handleMatch(m).text
+
         return re.sub(regex, handleMatch, self.source_text)
 
-def _markdownify_effect_text(move, effect_text):
+def _markdownify_effect_text(move, effect_text, language=None):
+    session = object_session(move)
+
     if effect_text is None:
         return effect_text
     effect_text = effect_text.replace(
@@ -79,7 +97,7 @@ def _markdownify_effect_text(move, effect_text):
         unicode(move.effect_chance),
     )
 
-    return MarkdownString(effect_text)
+    return MarkdownString(effect_text, session, language)
 
 class MoveEffectProperty(object):
     """Property that wraps move effects.  Used like this:
@@ -110,42 +128,22 @@ class MoveEffectPropertyMap(MoveEffectProperty):
         prop = getattr(obj.move_effect, self.effect_column)
         newdict = dict(prop)
         for key in newdict:
-            newdict[key] = _markdownify_effect_text(obj, newdict[key])
+            newdict[key] = _markdownify_effect_text(obj, newdict[key], key)
         return newdict
 
-class MarkdownColumn(sqlalchemy.types.TypeDecorator):
-    """Generic SQLAlchemy column type for Markdown text.
-
-    Do NOT use this for move effects!  They need to know what move they belong
-    to so they can fill in, e.g., effect chances.  Use the MoveEffectProperty
-    property class above.
-    """
-    impl = sqlalchemy.types.Unicode
-
-    def process_bind_param(self, value, dialect):
-        if value is None:
-            return None
-
-        if not isinstance(value, basestring):
-            # Can't assign, e.g., MarkdownString objects yet
-            raise NotImplementedError
-
-        return unicode(value)
-
-    def process_result_value(self, value, dialect):
-        if value is None:
-            return None
-
-        return MarkdownString(value)
-
 class PokedexLinkPattern(markdown.inlinepatterns.Pattern):
     """Matches [label]{category:target}.
+
+    Handles matches using factory
     """
     regex = ur'(?x) \[ ([^]]*) \] \{ ([-a-z0-9]+) : ([-a-z0-9]+) \}'
 
-    def __init__(self, extension):
+    def __init__(self, factory, string_language, game_language=None):
         markdown.inlinepatterns.Pattern.__init__(self, self.regex)
-        self.extension = extension
+        self.factory = factory
+        self.session = factory.session
+        self.string_language = string_language
+        self.game_language = game_language
 
     def handleMatch(self, m):
         from pokedex.db import tables, util
@@ -161,47 +159,54 @@ class PokedexLinkPattern(markdown.inlinepatterns.Pattern):
                 )[category]
         except KeyError:
             obj = name = target
-            url = self.extension.identifier_url(category, obj)
+            url = self.factory.identifier_url(category, obj)
         else:
-            session = self.extension.session
-            obj = util.get(self.extension.session, table, target)
-            url = self.extension.object_url(category, obj)
+            session = self.session
+            obj = util.get(self.session, table, target)
+            url = self.factory.object_url(category, obj)
+            url = url or self.factory.identifier_url(category, obj.identifier)
+            name = None
+            # Translations can be incomplete; in which case we want to use a
+            # fallback.
             if table in [tables.Type]:
                 # Type wants to be localized to the same language as the text
-                language = self.extension.language
-                name = None
-                try:
-                    name = obj.name_map[language]
-                except KeyError:
-                    pass
+                name = obj.name_map.get(self.string_language)
+            if not name and self.game_language:
+                name = obj.name_map.get(self.game_language)
             if not name:
                 name = obj.name
-            else:
-                name = obj.name
         if url:
-            el = self.extension.make_link(category, obj, url, label or name)
+            el = self.factory.make_link(category, obj, url, label or name)
         else:
             el = markdown.etree.Element('span')
             el.text = markdown.AtomicString(label or name)
         return el
 
-class PokedexLinkExtension(markdown.Extension):
-    """Plugs the [foo]{bar:baz} syntax into the markdown parser.
+class MarkdownLinkMaker(object):
+    """Creates Markdown extensions for handling links for the given session.
 
-    Subclases need to set the `session` attribute to the current session,
-    and `language` to the language of the strings.
-
-    To get links, subclasses must override object_url and/or identifier_url.
-    If these return None, <span>s are used instead of <a>.
+    There are two ways to customize the link handling: either override the
+    *_url methods in a subclass, or give them as arguments to get_extension
+    (or MarkdownString.as_html).
     """
-    language = None
-
-    def __init__(self):
-        markdown.Extension.__init__(self)
-        self.link_pattern = PokedexLinkPattern(self)
+    def __init__(self, session=None):
+        self.session = session
 
+    def get_extension(self, language=None, object_url=None, identifier_url=None,
+            make_link=None):
+        """Get a Markdown extension that handles links using the given language.
+        """
+        link_maker = self
+        class LinkExtension(markdown.Extension):
             def extendMarkdown(self, md, md_globals):
-        md.inlinePatterns['pokedex-link'] = self.link_pattern
+                self.identifier_url = identifier_url or link_maker.identifier_url
+                self.object_url = object_url or link_maker.object_url
+                self.make_link = make_link or link_maker.make_link
+                self.session = link_maker.session
+                pattern = PokedexLinkPattern(self, language)
+                md.inlinePatterns['pokedex-link'] = pattern
+
+        return LinkExtension()
 
     def make_link(self, category, obj, url, text):
         """Make an <a> element
@@ -216,8 +221,7 @@ class PokedexLinkExtension(markdown.Extension):
     def identifier_url(self, category, identifier):
         """Return the URL for the given {category:identifier} link
 
-        For ORM objects, object_url is used instead (but may fall back to
-        identifier_url).
+        For ORM objects, object_url is tried first
 
         Returns None by default, which causes <span> to be used in place of <a>
         """
@@ -226,16 +230,6 @@ class PokedexLinkExtension(markdown.Extension):
     def object_url(self, category, obj):
         """Return the URL for the ORM object obj
 
-        Calls identifier_url by default.
+        Returns None by default, which causes identifier_url to be used
         """
-        return self.identifier_url(category, obj.identifier)
-
-class ParametrizedLinkExtension(PokedexLinkExtension):
-    def __init__(self, session, object_url=None, identifier_url=None, language=None):
-        PokedexLinkExtension.__init__(self)
-        self.language = language
-        self.session = session
-        if object_url:
-            self.object_url = object_url
-        if identifier_url:
-            self.identifier_url = identifier_url
+        return None

pokedex/db/multilang.py

@@ -9,6 +9,8 @@ from sqlalchemy.schema import Column, ForeignKey, Table
 from sqlalchemy.sql.expression import and_, bindparam, select
 from sqlalchemy.types import Integer
 
+from pokedex.db import markdown
+
 def create_translation_table(_table_name, foreign_class, relation_name,
     language_class, relation_lazy='select', **kwargs):
     """Creates a table that represents some kind of data attached to the given
@@ -64,6 +66,9 @@ def create_translation_table(_table_name, foreign_class, relation_name,
     Pardon the naming disparity, but the grammar suffers otherwise.
 
     Modifying these directly is not likely to be a good idea.
+
+    For Markdown-formatted columns, `(column)_map` and `(column)` will give
+    Markdown objects.
     """
     # n.b.: language_class only exists for the sake of tests, which sometimes
     # want to create tables entirely separate from the pokedex metadata
@@ -132,9 +137,26 @@ def create_translation_table(_table_name, foreign_class, relation_name,
 
     # Add per-column proxies to the original class
     for name, column in kwitems:
+        string_getter = column.info.get('string_getter')
+        if string_getter:
+            def getset_factory(underlying_type, instance):
+                def getter(translations):
+                    text = getattr(translations, column.name)
+                    session = object_session(translations)
+                    language = translations.local_language
+                    return string_getter(text, session, language)
+                def setter(translations, value):
+                    # The string must be set on the Translation directly.
+                    raise AttributeError("Cannot set %s" % column.name)
+                return getter, setter
+            getset_factory = getset_factory
+        else:
+            getset_factory = None
+
         # Class.(column) -- accessor for the default language's value
         setattr(foreign_class, name,
-            association_proxy(local_relation_name, name))
+            association_proxy(local_relation_name, name,
+                    getset_factory=getset_factory))
 
         # Class.(column)_map -- accessor for the language dict
         # Need a custom creator since Translations doesn't have an init, and
@@ -145,7 +167,8 @@ def create_translation_table(_table_name, foreign_class, relation_name,
             setattr(row, name, value)
             return row
         setattr(foreign_class, name + '_map',
-            association_proxy(relation_name, name, creator=creator))
+            association_proxy(relation_name, name, creator=creator,
+                    getset_factory=getset_factory))
 
     # Add to the list of translation classes
     foreign_class.translation_classes.append(Translations)
@@ -164,6 +187,8 @@ class MultilangSession(Session):
         if 'default_language_id' in kwargs:
             self.default_language_id = kwargs.pop('default_language_id')
 
+        self.pokedex_link_maker = markdown.MarkdownLinkMaker(self)
+
         super(MultilangSession, self).__init__(*args, **kwargs)
 
     def execute(self, clause, params=None, *args, **kwargs):
@@ -189,3 +214,13 @@ class MultilangScopedSession(ScopedSession):
     @default_language_id.setter
     def default_language_id(self, new):
         self.registry().default_language_id = new
+
+    @property
+    def pokedex_link_maker(self):
+        """Passes the new link maker through to the current session.
+        """
+        return self.registry().pokedex_link_maker
+
+    @pokedex_link_maker.setter
+    def pokedex_link_maker(self, new):
+        self.registry().pokedex_link_maker = new

pokedex/db/tables.py

@@ -18,6 +18,10 @@ Columns have a info dictionary with these keys:
 - ripped: True for text that has been ripped from the games, and can be ripped
   again for new versions or languages
 
+ - string_getter: for translation columns, a function taking (text, session,
+    language) that is used for properties on the main table. Used for Markdown
+    text.
+
 See `pokedex.db.multilang` for how localizable text columns work.  The session
 classes in that module can be used to change the default language.
 """
@@ -124,10 +128,10 @@ create_translation_table('ability_names', Ability, 'names',
         info=dict(description="The name", format='plaintext', official=True, ripped=True)),
 )
 create_translation_table('ability_prose', Ability, 'prose',
-    effect = Column(markdown.MarkdownColumn(5120), nullable=True,
-        info=dict(description="A detailed description of this ability's effect", format='markdown')),
-    short_effect = Column(markdown.MarkdownColumn(512), nullable=True,
-        info=dict(description="A short summary of this ability's effect", format='markdown')),
+    effect = Column(Unicode(5120), nullable=True,
+        info=dict(description="A detailed description of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)),
+    short_effect = Column(Unicode(512), nullable=True,
+        info=dict(description="A short summary of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)),
 )
 
 class AbilityChangelog(TableBase):
@@ -142,8 +146,8 @@ class AbilityChangelog(TableBase):
         info=dict(description="The ID of the version group in which the ability changed"))
 
 create_translation_table('ability_changelog_prose', AbilityChangelog, 'prose',
-    effect = Column(markdown.MarkdownColumn(255), nullable=False,
-        info=dict(description="A description of the old behavior", format='markdown'))
+    effect = Column(Unicode(255), nullable=False,
+        info=dict(description="A description of the old behavior", format='markdown', string_getter=markdown.MarkdownString))
 )
 
 class AbilityFlavorText(TableBase):
@@ -503,10 +507,10 @@ create_translation_table('item_names', Item, 'names',
         info=dict(description="The name", format='plaintext', official=True, ripped=True)),
 )
 create_translation_table('item_prose', Item, 'prose',
-    short_effect = Column(markdown.MarkdownColumn(256), nullable=True,
-        info=dict(description="A short summary of the effect", format='markdown')),
-    effect = Column(markdown.MarkdownColumn(5120), nullable=True,
-        info=dict(description=u"Detailed description of the item's effect.", format='markdown')),
+    short_effect = Column(Unicode(256), nullable=True,
+        info=dict(description="A short summary of the effect", format='markdown', string_getter=markdown.MarkdownString)),
+    effect = Column(Unicode(5120), nullable=True,
+        info=dict(description=u"Detailed description of the item's effect.", format='markdown', string_getter=markdown.MarkdownString)),
 )
 create_translation_table('item_flavor_summaries', Item, 'flavor_summaries',
     flavor_summary = Column(Unicode(512), nullable=True,
@@ -803,7 +807,7 @@ class MoveEffect(TableBase):
 
 create_translation_table('move_effect_prose', MoveEffect, 'prose',
     short_effect = Column(Unicode(256), nullable=True,
-        info=dict(description="A short summary of the effect", format='plaintext')),
+        info=dict(description="A short summary of the effect", format='markdown')),
     effect = Column(Unicode(5120), nullable=True,
         info=dict(description="A detailed description of the effect", format='markdown')),
 )
@@ -825,8 +829,8 @@ class MoveEffectChangelog(TableBase):
     )
 
 create_translation_table('move_effect_changelog_prose', MoveEffectChangelog, 'prose',
-    effect = Column(markdown.MarkdownColumn(512), nullable=False,
-        info=dict(description="A description of the old behavior", format='markdown')),
+    effect = Column(Unicode(512), nullable=False,
+        info=dict(description="A description of the old behavior", format='markdown', string_getter=markdown.MarkdownString)),
 )
 
 class MoveFlag(TableBase):
@@ -854,8 +858,8 @@ create_translation_table('move_flag_type_prose', MoveFlagType, 'prose',
     relation_lazy='joined',
     name = Column(Unicode(32), nullable=True, index=True,
         info=dict(description="The name", format='plaintext', official=False)),
-    description = Column(markdown.MarkdownColumn(256), nullable=True,
-        info=dict(description="A short description of the flag", format='markdown')),
+    description = Column(Unicode(256), nullable=True,
+        info=dict(description="A short description of the flag", format='markdown', string_getter=markdown.MarkdownString)),
 )
 
 class MoveFlavorText(TableBase):
@@ -1355,8 +1359,8 @@ PokemonFormGroup.id = PokemonFormGroup.pokemon_id
 create_translation_table('pokemon_form_group_prose', PokemonFormGroup, 'prose',
     term = Column(Unicode(16), nullable=True,
         info=dict(description=u"The term for this Pokémon's forms, e.g. \"Cloak\" for Burmy or \"Forme\" for Deoxys.", official=True, format='plaintext')),
-    description = Column(markdown.MarkdownColumn(1024), nullable=True,
-        info=dict(description=u"Description of how the forms work", format='markdown')),
+    description = Column(Unicode(1024), nullable=True,
+        info=dict(description=u"Description of how the forms work", format='markdown', string_getter=markdown.MarkdownString)),
 )
 
 class PokemonFormPokeathlonStat(TableBase):

pokedex/tests/test_schema.py

@@ -180,17 +180,13 @@ def test_texts():
             if format is not None:
                 if format not in good_formats:
                     raise AssertionError(assert_text % column)
-                is_markdown = isinstance(column.type, markdown.MarkdownColumn)
-                if is_markdown and (format != 'markdown'):
-                    # Note: regular string with markdown syntax is allowed
-                    raise AssertionError('%s: markdown format/column type mismatch' % column)
                 if (format != 'identifier') and (column.name == 'identifier'):
                     raise AssertionError('%s: identifier column name/type mismatch' % column)
                 if column.info.get('official', None) and format not in 'gametext plaintext':
                     raise AssertionError('%s: official text with bad format' % column)
                 text_columns.append(column)
             else:
-                if isinstance(column.type, (markdown.MarkdownColumn, tables.Unicode)):
+                if isinstance(column.type, tables.Unicode):
                     raise AssertionError('%s: text column without format' % column)
             if column.name == 'name' and format != 'plaintext':
                 raise AssertionError('%s: non-plaintext name' % column)

pokedex/tests/test_strings.py

@@ -2,7 +2,7 @@
 
 from nose.tools import *
 
-from pokedex.db import tables, connect
+from pokedex.db import tables, connect, util, markdown
 
 class TestStrings(object):
     def setup(self):
@@ -83,5 +83,22 @@ class TestStrings(object):
                 identifier=u"thunderbolt").one()
         language = self.connection.query(tables.Language).filter_by(
                 identifier=u"en").one()
-        assert '10%' in move.effect.as_text
-        assert '10%' in move.effect_map[language].as_text
+        assert '10%' in move.effect.as_text()
+        assert '10%' in move.effect_map[language].as_text()
+        assert '10%' in move.effect.as_html()
+        assert '10%' in move.effect_map[language].as_html()
+        assert '10%' in unicode(move.effect)
+        assert '10%' in unicode(move.effect_map[language])
+        assert '10%' in move.effect.__html__()
+        assert '10%' in move.effect_map[language].__html__()
+
+    def test_markdown_string(self):
+        en = util.get(self.connection, tables.Language, 'en')
+        md = markdown.MarkdownString('[]{move:thunderbolt} [paralyzes]{mechanic:paralysis}', self.connection, en)
+        assert unicode(md) == 'Thunderbolt paralyzes'
+        assert md.as_html() == '<p><span>Thunderbolt</span> <span>paralyzes</span></p>'
+        assert md.as_html(object_url=lambda category, obj: "%s/%s" % (category, obj.identifier)) == (
+                '<p><a href="move/thunderbolt">Thunderbolt</a> <span>paralyzes</span></p>')
+        print md.as_html(identifier_url=lambda category, ident: "%s/%s" % (category, ident))
+        assert md.as_html(identifier_url=lambda category, ident: "%s/%s" % (category, ident)) == (
+                '<p><a href="move/thunderbolt">Thunderbolt</a> <a href="mechanic/paralysis">paralyzes</a></p>')