Dynamically mangle long table names for Oracle; Unicode → UnicodeText.

This commit is contained in:
Epithumia 2013-12-18 14:32:13 +01:00 committed by Lynn "Zhorken" Vaughan
parent bf2c79879d
commit 7b2743be75
6 changed files with 84 additions and 11 deletions

View file

@ -108,9 +108,9 @@ For example, you can get a list of all pokémon species, sorted by their
Charmander Charmander
Charmeleon Charmeleon
... ...
Keldeo Xerneas
Meloetta Yveltal
Genesect Zygarde
Or to order by :attr:`~pokedex.db.tables.PokemonSpecies.name`: Or to order by :attr:`~pokedex.db.tables.PokemonSpecies.name`:
@ -123,7 +123,7 @@ Or to order by :attr:`~pokedex.db.tables.PokemonSpecies.name`:
Abomasnow Abomasnow
... ...
Zweilous Zygarde
Filtering Filtering
@ -171,7 +171,7 @@ example:
Seed Flare (120) Seed Flare (120)
Solar Beam (120) Solar Beam (120)
Wood Hammer (120) Wood Hammer (120)
Leaf Storm (140) Leaf Storm (130)
Frenzy Plant (150) Frenzy Plant (150)
That concludes our brief tutorial. That concludes our brief tutorial.

View file

@ -38,6 +38,13 @@ def connect(uri=None, session_args={}, engine_args={}, engine_prefix=''):
table.kwargs['mysql_engine'] = 'InnoDB' table.kwargs['mysql_engine'] = 'InnoDB'
table.kwargs['mysql_charset'] = 'utf8' table.kwargs['mysql_charset'] = 'utf8'
### Do some fixery for Oracle
if uri.startswith('oracle:') or uri.startswith('oracle+cx_oracle:'):
# Oracle requires auto_setinputsizes=False (or at least a special
# set of exclusions from it, which I don't know)
if 'auto_setinputsizes' not in uri:
uri += '?auto_setinputsizes=FALSE'
### Connect ### Connect
engine_args[engine_prefix + 'url'] = uri engine_args[engine_prefix + 'url'] = uri
engine = engine_from_config(engine_args, prefix=engine_prefix) engine = engine_from_config(engine_args, prefix=engine_prefix)

View file

@ -11,6 +11,7 @@ import pokedex
from pokedex.db import metadata, tables, translations from pokedex.db import metadata, tables, translations
from pokedex.defaults import get_default_csv_dir from pokedex.defaults import get_default_csv_dir
from pokedex.db.dependencies import find_dependent_tables from pokedex.db.dependencies import find_dependent_tables
from pokedex.db.oracle import rewrite_long_table_names
def _get_table_names(metadata, patterns): def _get_table_names(metadata, patterns):
@ -138,6 +139,14 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s
table_names = _get_table_names(metadata, tables) table_names = _get_table_names(metadata, tables)
table_objs = [metadata.tables[name] for name in table_names] table_objs = [metadata.tables[name] for name in table_names]
# Oracle fixery, load needs short names
# flag for oracle stuff
oranames = (session.connection().dialect.name == 'oracle')
if oranames:
# Shorten table names, Oracle limits table and column names to 30 chars
# Make a dictionary to match old<->new names
oradict = rewrite_long_table_names()
if recursive: if recursive:
table_objs.extend(find_dependent_tables(table_objs)) table_objs.extend(find_dependent_tables(table_objs))
@ -181,9 +190,11 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s
insert_stmt = table_obj.insert() insert_stmt = table_obj.insert()
print_start(table_name) print_start(table_name)
try: try:
csvpath = "%s/%s.csv" % (directory, table_name) csvpath = "%s/%s.csv" % (directory, table_name)
# In oracle mode, use the original names instead of current
if oranames:
csvpath = "%s/%s.csv" % (directory, oradict[table_name])
csvfile = open(csvpath, 'rb') csvfile = open(csvpath, 'rb')
except IOError: except IOError:
# File doesn't exist; don't load anything! # File doesn't exist; don't load anything!
@ -250,6 +261,9 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s
for column_name, value in zip(column_names, csvs): for column_name, value in zip(column_names, csvs):
column = table_obj.c[column_name] column = table_obj.c[column_name]
# Oracle treats empty strings as NULL
if not column.nullable and value == '' and oranames:
value = ' '
if column.nullable and value == '': if column.nullable and value == '':
# Empty string in a nullable column really means NULL # Empty string in a nullable column really means NULL
value = None value = None
@ -369,6 +383,11 @@ def dump(session, tables=[], directory=None, verbose=False, langs=['en']):
table_names = _get_table_names(metadata, tables) table_names = _get_table_names(metadata, tables)
table_names.sort() table_names.sort()
# Oracle fixery : read from short table names, dump long names
oranames = (session.connection().dialect.name == 'oracle')
if oranames:
# Make a dictionary to match old<->new names
oradict = rewrite_long_table_names()
for table_name in table_names: for table_name in table_names:
print_start(table_name) print_start(table_name)
@ -376,6 +395,10 @@ def dump(session, tables=[], directory=None, verbose=False, langs=['en']):
writer = csv.writer(open("%s/%s.csv" % (directory, table_name), 'wb'), writer = csv.writer(open("%s/%s.csv" % (directory, table_name), 'wb'),
lineterminator='\n') lineterminator='\n')
# In oracle mode, use the original names instead of current
if oranames:
writer = csv.writer(open("%s/%s.csv" % (directory, oradict[table_name]), 'wb'),
lineterminator='\n')
columns = [col.name for col in table.columns] columns = [col.name for col in table.columns]
# For name tables, dump rows for official languages, as well as # For name tables, dump rows for official languages, as well as

41
pokedex/db/oracle.py Normal file
View file

@ -0,0 +1,41 @@
from pokedex.db import metadata
### Helper functions for oracle
def rewrite_long_table_names():
"""Modifies the table names to disenvowel long table names.
Takes the metadata from memory or uses the imported one.
Returns a dictionary matching short names -> long names.
"""
# Load table names from metadata
t_names = metadata.tables.keys()
table_names = list(t_names)
table_objs = [metadata.tables[name] for name in table_names]
# Prepare a dictionary to match old<->new names
dictionary = {}
# Shorten table names, Oracle limits table and column names to 30 chars
for table in table_objs:
table._orginal_name = table.name[:]
dictionary[table.name]=table._orginal_name
if len(table._orginal_name) > 30:
for letter in ['a', 'e', 'i', 'o', 'u', 'y']:
table.name=table.name.replace(letter,'')
dictionary[table.name]=table._orginal_name
return dictionary
def restore_long_table_names(metadata,dictionary):
"""Modifies the table names to restore the long-naming.
`metadata`
The metadata to restore.
`dictionary`
The dictionary matching short name -> long name.
"""
for table in metadata.tables.values():
table.name = dictionary[table.name]

View file

@ -38,7 +38,7 @@ from sqlalchemy.orm.session import Session
from sqlalchemy.orm.interfaces import AttributeExtension from sqlalchemy.orm.interfaces import AttributeExtension
from sqlalchemy.sql import and_, or_ from sqlalchemy.sql import and_, or_
from sqlalchemy.schema import ColumnDefault from sqlalchemy.schema import ColumnDefault
from sqlalchemy.types import Boolean, Enum, Integer, SmallInteger, Unicode from sqlalchemy.types import Boolean, Enum, Integer, SmallInteger, Unicode, UnicodeText
from pokedex.db import markdown, multilang from pokedex.db import markdown, multilang
@ -130,7 +130,7 @@ create_translation_table('ability_names', Ability, 'names',
info=dict(description="The name", format='plaintext', official=True, ripped=True)), info=dict(description="The name", format='plaintext', official=True, ripped=True)),
) )
create_translation_table('ability_prose', Ability, 'prose', create_translation_table('ability_prose', Ability, 'prose',
effect = Column(Unicode(4000), nullable=True, effect = Column(UnicodeText(), nullable=True,
info=dict(description="A detailed description of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)), info=dict(description="A detailed description of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)),
short_effect = Column(Unicode(512), nullable=True, short_effect = Column(Unicode(512), nullable=True,
info=dict(description="A short summary of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)), info=dict(description="A short summary of this ability's effect", format='markdown', string_getter=markdown.MarkdownString)),
@ -920,7 +920,7 @@ create_translation_table('item_names', Item, 'names',
create_translation_table('item_prose', Item, 'prose', create_translation_table('item_prose', Item, 'prose',
short_effect = Column(Unicode(256), nullable=True, short_effect = Column(Unicode(256), nullable=True,
info=dict(description="A short summary of the effect", format='markdown', string_getter=markdown.MarkdownString)), info=dict(description="A short summary of the effect", format='markdown', string_getter=markdown.MarkdownString)),
effect = Column(Unicode(4000), nullable=True, effect = Column(UnicodeText(), nullable=True,
info=dict(description=u"Detailed description of the item's effect.", format='markdown', string_getter=markdown.MarkdownString)), info=dict(description=u"Detailed description of the item's effect.", format='markdown', string_getter=markdown.MarkdownString)),
) )
create_translation_table('item_flavor_summaries', Item, 'flavor_summaries', create_translation_table('item_flavor_summaries', Item, 'flavor_summaries',
@ -1219,7 +1219,7 @@ class MoveEffect(TableBase):
create_translation_table('move_effect_prose', MoveEffect, 'prose', create_translation_table('move_effect_prose', MoveEffect, 'prose',
short_effect = Column(Unicode(256), nullable=True, short_effect = Column(Unicode(256), nullable=True,
info=dict(description="A short summary of the effect", format='markdown')), info=dict(description="A short summary of the effect", format='markdown')),
effect = Column(Unicode(4000), nullable=True, effect = Column(UnicodeText(), nullable=True,
info=dict(description="A detailed description of the effect", format='markdown')), info=dict(description="A detailed description of the effect", format='markdown')),
) )

View file

@ -56,6 +56,8 @@ def column_type_str(column):
return 'bool' return 'bool'
if type(type_) == types.Unicode: if type(type_) == types.Unicode:
return u'unicode %s' % column.info['format'] return u'unicode %s' % column.info['format']
if type(type_) == types.UnicodeText:
return u'unicode %s' % column.info['format']
if type(type_) == types.Enum: if type(type_) == types.Enum:
return 'enum: [%s]' % ', '.join(type_.enums) return 'enum: [%s]' % ', '.join(type_.enums)
if type(type_) == markdown.MarkdownColumn: if type(type_) == markdown.MarkdownColumn: