diff --git a/doc/usage.rst b/doc/usage.rst index 2d3c08b..50b39fc 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -108,9 +108,9 @@ For example, you can get a list of all pokémon species, sorted by their Charmander Charmeleon ... - Keldeo - Meloetta - Genesect + Xerneas + Yveltal + Zygarde Or to order by :attr:`~pokedex.db.tables.PokemonSpecies.name`: @@ -123,7 +123,7 @@ Or to order by :attr:`~pokedex.db.tables.PokemonSpecies.name`: Abomasnow ... - Zweilous + Zygarde Filtering @@ -169,9 +169,9 @@ example: Petal Dance (120) Power Whip (120) Seed Flare (120) - SolarBeam (120) + Solar Beam (120) Wood Hammer (120) - Leaf Storm (140) + Leaf Storm (130) Frenzy Plant (150) That concludes our brief tutorial. diff --git a/pokedex/data/csv/item_categories.csv b/pokedex/data/csv/item_categories.csv index 75e9d24..d2e2082 100644 --- a/pokedex/data/csv/item_categories.csv +++ b/pokedex/data/csv/item_categories.csv @@ -23,7 +23,7 @@ id,pocket_id,identifier 22,8,plot-advancement 23,8,unused 24,1,loot -25,6, +25,6,all-mail 26,2,vitamins 27,2,healing 28,2,pp-recovery @@ -34,7 +34,7 @@ id,pocket_id,identifier 34,3,standard-balls 35,1,dex-completion 36,1,scarves -37,4, +37,4,all-machines 38,7,flutes 39,3,apricorn-balls 40,8,apricorn-box diff --git a/pokedex/data/csv/item_category_prose.csv b/pokedex/data/csv/item_category_prose.csv index ead8c30..6ac4725 100644 --- a/pokedex/data/csv/item_category_prose.csv +++ b/pokedex/data/csv/item_category_prose.csv @@ -23,7 +23,7 @@ item_category_id,local_language_id,name 22,9,Plot advancement 23,9,Unused 24,9,Loot -25,9, +25,9,All mail 26,9,Vitamins 27,9,Healing 28,9,PP recovery @@ -34,7 +34,7 @@ item_category_id,local_language_id,name 34,9,Standard balls 35,9,Dex completion 36,9,Scarves -37,9, +37,9,All machines 38,9,Flutes 39,9,Apricorn balls 40,9,Apricorn Box diff --git a/pokedex/db/__init__.py b/pokedex/db/__init__.py index 464629d..3e8ecb1 100644 --- a/pokedex/db/__init__.py +++ b/pokedex/db/__init__.py @@ -25,7 +25,7 @@ def connect(uri=None, session_args={}, engine_args={}, engine_prefix=''): if uri is None: uri = get_default_db_uri() - ### Do some fixery for MySQL + ### Do some fixery for specific RDBMSes if uri.startswith('mysql:'): # MySQL uses latin1 for connections by default even if the server is # otherwise oozing with utf8; charset fixes this @@ -37,6 +37,11 @@ def connect(uri=None, session_args={}, engine_args={}, engine_prefix=''): for table in metadata.tables.values(): table.kwargs['mysql_engine'] = 'InnoDB' table.kwargs['mysql_charset'] = 'utf8' + elif uri.startswith(('oracle:', 'oracle+cx_oracle:')): + # Oracle requires auto_setinputsizes=False (or at least a special + # set of exclusions from it, which I don't know) + if 'auto_setinputsizes' not in uri: + uri += '?auto_setinputsizes=FALSE' ### Connect engine_args[engine_prefix + 'url'] = uri diff --git a/pokedex/db/load.py b/pokedex/db/load.py index d241519..01859a2 100644 --- a/pokedex/db/load.py +++ b/pokedex/db/load.py @@ -11,6 +11,7 @@ import pokedex from pokedex.db import metadata, tables, translations from pokedex.defaults import get_default_csv_dir from pokedex.db.dependencies import find_dependent_tables +from pokedex.db.oracle import rewrite_long_table_names def _get_table_names(metadata, patterns): @@ -143,6 +144,11 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s table_objs = sqlalchemy.sql.util.sort_tables(table_objs) + # Limit table names to 30 characters for Oracle + oracle = (session.connection().dialect.name == 'oracle') + if oracle: + rewrite_long_table_names() + # SQLite speed tweaks if not safe and session.connection().dialect.name == 'sqlite': session.connection().execute("PRAGMA synchronous=OFF") @@ -177,7 +183,11 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s # Okay, run through the tables and actually load the data now for table_obj in table_objs: - table_name = table_obj.name + if oracle: + table_name = table_obj._original_name + else: + table_name = table_obj.name + insert_stmt = table_obj.insert() print_start(table_name) @@ -369,13 +379,23 @@ def dump(session, tables=[], directory=None, verbose=False, langs=['en']): table_names = _get_table_names(metadata, tables) table_names.sort() + # Oracle needs to dump from tables with shortened names to csvs with the + # usual names + oracle = (session.connection().dialect.name == 'oracle') + if oracle: + rewrite_long_table_names() for table_name in table_names: print_start(table_name) table = metadata.tables[table_name] - writer = csv.writer(open("%s/%s.csv" % (directory, table_name), 'wb'), - lineterminator='\n') + if oracle: + filename = '%s/%s.csv' % (directory, table._original_name) + else: + filename = '%s/%s.csv' % (directory, table_name) + + writer = csv.writer(open(filename, 'wb'), lineterminator='\n') + columns = [col.name for col in table.columns] # For name tables, dump rows for official languages, as well as diff --git a/pokedex/db/oracle.py b/pokedex/db/oracle.py new file mode 100644 index 0000000..ad9fd11 --- /dev/null +++ b/pokedex/db/oracle.py @@ -0,0 +1,26 @@ +from pokedex.db import metadata + +### Helper functions for oracle +def rewrite_long_table_names(): + """Modifies the table names to disenvowel long table names. + Takes the metadata from memory or uses the imported one. + + Returns a dictionary matching short names -> long names. + """ + + # Load tables from metadata + table_objs = metadata.tables.values() + + # Shorten table names, Oracle limits table and column names to 30 chars + for table in table_objs: + table._original_name = table.name + + if len(table.name) > 30: + for letter in 'aeiouy': + table.name = table.name.replace(letter, '') + +def restore_long_table_names(): + """Modifies the table names to restore the long-naming.""" + for table in metadata.tables.values(): + table.name = table._original_name + del table._original_name diff --git a/pokedex/db/tables.py b/pokedex/db/tables.py index b24ea86..9c85a36 100644 --- a/pokedex/db/tables.py +++ b/pokedex/db/tables.py @@ -38,7 +38,7 @@ from sqlalchemy.orm.session import Session from sqlalchemy.orm.interfaces import AttributeExtension from sqlalchemy.sql import and_, or_ from sqlalchemy.schema import ColumnDefault -from sqlalchemy.types import Boolean, Enum, Integer, SmallInteger, Unicode +from sqlalchemy.types import Boolean, Enum, Integer, SmallInteger, Unicode, UnicodeText from pokedex.db import markdown, multilang @@ -130,7 +130,7 @@ create_translation_table('ability_names', Ability, 'names', info=dict(description="The name", format='plaintext', official=True, ripped=True)), ) create_translation_table('ability_prose', Ability, 'prose', - effect = Column(Unicode(4000), nullable=True, + effect = Column(UnicodeText(), nullable=True, info=dict(description="A detailed description of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)), short_effect = Column(Unicode(512), nullable=True, info=dict(description="A short summary of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)), @@ -920,7 +920,7 @@ create_translation_table('item_names', Item, 'names', create_translation_table('item_prose', Item, 'prose', short_effect = Column(Unicode(256), nullable=True, info=dict(description="A short summary of the effect", format='markdown', string_getter=markdown.MarkdownString)), - effect = Column(Unicode(4000), nullable=True, + effect = Column(UnicodeText(), nullable=True, info=dict(description=u"Detailed description of the item's effect.", format='markdown', string_getter=markdown.MarkdownString)), ) create_translation_table('item_flavor_summaries', Item, 'flavor_summaries', @@ -1062,7 +1062,7 @@ class LocationArea(TableBase): create_translation_table('location_area_prose', LocationArea, 'prose', relation_lazy='joined', - name = Column(Unicode(64), nullable=False, index=True, + name = Column(Unicode(64), nullable=True, index=True, info=dict(description="The name", format='plaintext', official=False)), ) @@ -1219,7 +1219,7 @@ class MoveEffect(TableBase): create_translation_table('move_effect_prose', MoveEffect, 'prose', short_effect = Column(Unicode(256), nullable=True, info=dict(description="A short summary of the effect", format='markdown')), - effect = Column(Unicode(4000), nullable=True, + effect = Column(UnicodeText(), nullable=True, info=dict(description="A detailed description of the effect", format='markdown')), ) diff --git a/pokedex/doc/tabledoc.py b/pokedex/doc/tabledoc.py index 172a454..755303b 100644 --- a/pokedex/doc/tabledoc.py +++ b/pokedex/doc/tabledoc.py @@ -56,6 +56,8 @@ def column_type_str(column): return 'bool' if type(type_) == types.Unicode: return u'unicode – %s' % column.info['format'] + if type(type_) == types.UnicodeText: + return u'unicode – %s' % column.info['format'] if type(type_) == types.Enum: return 'enum: [%s]' % ', '.join(type_.enums) if type(type_) == markdown.MarkdownColumn: