Merge branch 'oracle'. #109

This commit is contained in:
Lynn "Zhorken" Vaughan 2014-02-21 17:51:37 -05:00
commit 0c8aadc53c
8 changed files with 72 additions and 19 deletions

View file

@ -108,9 +108,9 @@ For example, you can get a list of all pokémon species, sorted by their
Charmander Charmander
Charmeleon Charmeleon
... ...
Keldeo Xerneas
Meloetta Yveltal
Genesect Zygarde
Or to order by :attr:`~pokedex.db.tables.PokemonSpecies.name`: Or to order by :attr:`~pokedex.db.tables.PokemonSpecies.name`:
@ -123,7 +123,7 @@ Or to order by :attr:`~pokedex.db.tables.PokemonSpecies.name`:
Abomasnow Abomasnow
... ...
Zweilous Zygarde
Filtering Filtering
@ -169,9 +169,9 @@ example:
Petal Dance (120) Petal Dance (120)
Power Whip (120) Power Whip (120)
Seed Flare (120) Seed Flare (120)
SolarBeam (120) Solar Beam (120)
Wood Hammer (120) Wood Hammer (120)
Leaf Storm (140) Leaf Storm (130)
Frenzy Plant (150) Frenzy Plant (150)
That concludes our brief tutorial. That concludes our brief tutorial.

View file

@ -23,7 +23,7 @@ id,pocket_id,identifier
22,8,plot-advancement 22,8,plot-advancement
23,8,unused 23,8,unused
24,1,loot 24,1,loot
25,6, 25,6,all-mail
26,2,vitamins 26,2,vitamins
27,2,healing 27,2,healing
28,2,pp-recovery 28,2,pp-recovery
@ -34,7 +34,7 @@ id,pocket_id,identifier
34,3,standard-balls 34,3,standard-balls
35,1,dex-completion 35,1,dex-completion
36,1,scarves 36,1,scarves
37,4, 37,4,all-machines
38,7,flutes 38,7,flutes
39,3,apricorn-balls 39,3,apricorn-balls
40,8,apricorn-box 40,8,apricorn-box

1 id pocket_id identifier
23 22 8 plot-advancement
24 23 8 unused
25 24 1 loot
26 25 6 all-mail
27 26 2 vitamins
28 27 2 healing
29 28 2 pp-recovery
34 34 3 standard-balls
35 35 1 dex-completion
36 36 1 scarves
37 37 4 all-machines
38 38 7 flutes
39 39 3 apricorn-balls
40 40 8 apricorn-box

View file

@ -23,7 +23,7 @@ item_category_id,local_language_id,name
22,9,Plot advancement 22,9,Plot advancement
23,9,Unused 23,9,Unused
24,9,Loot 24,9,Loot
25,9, 25,9,All mail
26,9,Vitamins 26,9,Vitamins
27,9,Healing 27,9,Healing
28,9,PP recovery 28,9,PP recovery
@ -34,7 +34,7 @@ item_category_id,local_language_id,name
34,9,Standard balls 34,9,Standard balls
35,9,Dex completion 35,9,Dex completion
36,9,Scarves 36,9,Scarves
37,9, 37,9,All machines
38,9,Flutes 38,9,Flutes
39,9,Apricorn balls 39,9,Apricorn balls
40,9,Apricorn Box 40,9,Apricorn Box

1 item_category_id local_language_id name
23 22 9 Plot advancement
24 23 9 Unused
25 24 9 Loot
26 25 9 All mail
27 26 9 Vitamins
28 27 9 Healing
29 28 9 PP recovery
34 34 9 Standard balls
35 35 9 Dex completion
36 36 9 Scarves
37 37 9 All machines
38 38 9 Flutes
39 39 9 Apricorn balls
40 40 9 Apricorn Box

View file

@ -25,7 +25,7 @@ def connect(uri=None, session_args={}, engine_args={}, engine_prefix=''):
if uri is None: if uri is None:
uri = get_default_db_uri() uri = get_default_db_uri()
### Do some fixery for MySQL ### Do some fixery for specific RDBMSes
if uri.startswith('mysql:'): if uri.startswith('mysql:'):
# MySQL uses latin1 for connections by default even if the server is # MySQL uses latin1 for connections by default even if the server is
# otherwise oozing with utf8; charset fixes this # otherwise oozing with utf8; charset fixes this
@ -37,6 +37,11 @@ def connect(uri=None, session_args={}, engine_args={}, engine_prefix=''):
for table in metadata.tables.values(): for table in metadata.tables.values():
table.kwargs['mysql_engine'] = 'InnoDB' table.kwargs['mysql_engine'] = 'InnoDB'
table.kwargs['mysql_charset'] = 'utf8' table.kwargs['mysql_charset'] = 'utf8'
elif uri.startswith(('oracle:', 'oracle+cx_oracle:')):
# Oracle requires auto_setinputsizes=False (or at least a special
# set of exclusions from it, which I don't know)
if 'auto_setinputsizes' not in uri:
uri += '?auto_setinputsizes=FALSE'
### Connect ### Connect
engine_args[engine_prefix + 'url'] = uri engine_args[engine_prefix + 'url'] = uri

View file

@ -11,6 +11,7 @@ import pokedex
from pokedex.db import metadata, tables, translations from pokedex.db import metadata, tables, translations
from pokedex.defaults import get_default_csv_dir from pokedex.defaults import get_default_csv_dir
from pokedex.db.dependencies import find_dependent_tables from pokedex.db.dependencies import find_dependent_tables
from pokedex.db.oracle import rewrite_long_table_names
def _get_table_names(metadata, patterns): def _get_table_names(metadata, patterns):
@ -143,6 +144,11 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s
table_objs = sqlalchemy.sql.util.sort_tables(table_objs) table_objs = sqlalchemy.sql.util.sort_tables(table_objs)
# Limit table names to 30 characters for Oracle
oracle = (session.connection().dialect.name == 'oracle')
if oracle:
rewrite_long_table_names()
# SQLite speed tweaks # SQLite speed tweaks
if not safe and session.connection().dialect.name == 'sqlite': if not safe and session.connection().dialect.name == 'sqlite':
session.connection().execute("PRAGMA synchronous=OFF") session.connection().execute("PRAGMA synchronous=OFF")
@ -177,7 +183,11 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s
# Okay, run through the tables and actually load the data now # Okay, run through the tables and actually load the data now
for table_obj in table_objs: for table_obj in table_objs:
table_name = table_obj.name if oracle:
table_name = table_obj._original_name
else:
table_name = table_obj.name
insert_stmt = table_obj.insert() insert_stmt = table_obj.insert()
print_start(table_name) print_start(table_name)
@ -369,13 +379,23 @@ def dump(session, tables=[], directory=None, verbose=False, langs=['en']):
table_names = _get_table_names(metadata, tables) table_names = _get_table_names(metadata, tables)
table_names.sort() table_names.sort()
# Oracle needs to dump from tables with shortened names to csvs with the
# usual names
oracle = (session.connection().dialect.name == 'oracle')
if oracle:
rewrite_long_table_names()
for table_name in table_names: for table_name in table_names:
print_start(table_name) print_start(table_name)
table = metadata.tables[table_name] table = metadata.tables[table_name]
writer = csv.writer(open("%s/%s.csv" % (directory, table_name), 'wb'), if oracle:
lineterminator='\n') filename = '%s/%s.csv' % (directory, table._original_name)
else:
filename = '%s/%s.csv' % (directory, table_name)
writer = csv.writer(open(filename, 'wb'), lineterminator='\n')
columns = [col.name for col in table.columns] columns = [col.name for col in table.columns]
# For name tables, dump rows for official languages, as well as # For name tables, dump rows for official languages, as well as

26
pokedex/db/oracle.py Normal file
View file

@ -0,0 +1,26 @@
from pokedex.db import metadata
### Helper functions for oracle
def rewrite_long_table_names():
"""Modifies the table names to disenvowel long table names.
Takes the metadata from memory or uses the imported one.
Returns a dictionary matching short names -> long names.
"""
# Load tables from metadata
table_objs = metadata.tables.values()
# Shorten table names, Oracle limits table and column names to 30 chars
for table in table_objs:
table._original_name = table.name
if len(table.name) > 30:
for letter in 'aeiouy':
table.name = table.name.replace(letter, '')
def restore_long_table_names():
"""Modifies the table names to restore the long-naming."""
for table in metadata.tables.values():
table.name = table._original_name
del table._original_name

View file

@ -38,7 +38,7 @@ from sqlalchemy.orm.session import Session
from sqlalchemy.orm.interfaces import AttributeExtension from sqlalchemy.orm.interfaces import AttributeExtension
from sqlalchemy.sql import and_, or_ from sqlalchemy.sql import and_, or_
from sqlalchemy.schema import ColumnDefault from sqlalchemy.schema import ColumnDefault
from sqlalchemy.types import Boolean, Enum, Integer, SmallInteger, Unicode from sqlalchemy.types import Boolean, Enum, Integer, SmallInteger, Unicode, UnicodeText
from pokedex.db import markdown, multilang from pokedex.db import markdown, multilang
@ -130,7 +130,7 @@ create_translation_table('ability_names', Ability, 'names',
info=dict(description="The name", format='plaintext', official=True, ripped=True)), info=dict(description="The name", format='plaintext', official=True, ripped=True)),
) )
create_translation_table('ability_prose', Ability, 'prose', create_translation_table('ability_prose', Ability, 'prose',
effect = Column(Unicode(4000), nullable=True, effect = Column(UnicodeText(), nullable=True,
info=dict(description="A detailed description of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)), info=dict(description="A detailed description of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)),
short_effect = Column(Unicode(512), nullable=True, short_effect = Column(Unicode(512), nullable=True,
info=dict(description="A short summary of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)), info=dict(description="A short summary of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)),
@ -920,7 +920,7 @@ create_translation_table('item_names', Item, 'names',
create_translation_table('item_prose', Item, 'prose', create_translation_table('item_prose', Item, 'prose',
short_effect = Column(Unicode(256), nullable=True, short_effect = Column(Unicode(256), nullable=True,
info=dict(description="A short summary of the effect", format='markdown', string_getter=markdown.MarkdownString)), info=dict(description="A short summary of the effect", format='markdown', string_getter=markdown.MarkdownString)),
effect = Column(Unicode(4000), nullable=True, effect = Column(UnicodeText(), nullable=True,
info=dict(description=u"Detailed description of the item's effect.", format='markdown', string_getter=markdown.MarkdownString)), info=dict(description=u"Detailed description of the item's effect.", format='markdown', string_getter=markdown.MarkdownString)),
) )
create_translation_table('item_flavor_summaries', Item, 'flavor_summaries', create_translation_table('item_flavor_summaries', Item, 'flavor_summaries',
@ -1062,7 +1062,7 @@ class LocationArea(TableBase):
create_translation_table('location_area_prose', LocationArea, 'prose', create_translation_table('location_area_prose', LocationArea, 'prose',
relation_lazy='joined', relation_lazy='joined',
name = Column(Unicode(64), nullable=False, index=True, name = Column(Unicode(64), nullable=True, index=True,
info=dict(description="The name", format='plaintext', official=False)), info=dict(description="The name", format='plaintext', official=False)),
) )
@ -1219,7 +1219,7 @@ class MoveEffect(TableBase):
create_translation_table('move_effect_prose', MoveEffect, 'prose', create_translation_table('move_effect_prose', MoveEffect, 'prose',
short_effect = Column(Unicode(256), nullable=True, short_effect = Column(Unicode(256), nullable=True,
info=dict(description="A short summary of the effect", format='markdown')), info=dict(description="A short summary of the effect", format='markdown')),
effect = Column(Unicode(4000), nullable=True, effect = Column(UnicodeText(), nullable=True,
info=dict(description="A detailed description of the effect", format='markdown')), info=dict(description="A detailed description of the effect", format='markdown')),
) )

View file

@ -56,6 +56,8 @@ def column_type_str(column):
return 'bool' return 'bool'
if type(type_) == types.Unicode: if type(type_) == types.Unicode:
return u'unicode %s' % column.info['format'] return u'unicode %s' % column.info['format']
if type(type_) == types.UnicodeText:
return u'unicode %s' % column.info['format']
if type(type_) == types.Enum: if type(type_) == types.Enum:
return 'enum: [%s]' % ', '.join(type_.enums) return 'enum: [%s]' % ', '.join(type_.enums)
if type(type_) == markdown.MarkdownColumn: if type(type_) == markdown.MarkdownColumn: