veniasilente/veekun_pokedex
1
0
Fork 0
mirror of https://github.com/veekun/pokedex.git synced 2024-08-20 18:16:34 +00:00
Code Issues Projects Releases Wiki Activity

Speed tweaks for pokedex load in SQLite

Browse source
This commit is contained in:
Petr Viktorin 2011-01-08 23:21:44 +01:00 committed by Eevee
parent 570719499d
commit 497ba412b0
2 changed files with 17 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
pokedex/__init__.py
View file

@ -137,7 +137,8 @@ def command_load(*args):
pokedex.db.load.load(session, directory=options.directory, pokedex.db.load.load(session, directory=options.directory,
drop_tables=options.drop_tables, drop_tables=options.drop_tables,
tables=tables, tables=tables,
verbose=options.verbose) verbose=options.verbose,
safe=False)
def command_reindex(*args): def command_reindex(*args):
parser = get_parser(verbose=True) parser = get_parser(verbose=True)
@ -158,7 +159,8 @@ def command_setup(*args):
session = get_session(options) session = get_session(options)
get_csv_directory(options) get_csv_directory(options)
pokedex.db.load.load(session, directory=None, drop_tables=True, pokedex.db.load.load(session, directory=None, drop_tables=True,
verbose=options.verbose) verbose=options.verbose,
safe=False)
lookup = get_lookup(options, session=session, recreate=True) lookup = get_lookup(options, session=session, recreate=True)

14
pokedex/db/load.py
View file

@ -95,7 +95,7 @@ def _get_verbose_prints(verbose):
return print_start, print_status, print_done return print_start, print_status, print_done
def load(session, tables=[], directory=None, drop_tables=False, verbose=False): def load(session, tables=[], directory=None, drop_tables=False, verbose=False, safe=True):
"""Load data from CSV files into the given database session. """Load data from CSV files into the given database session.
Tables are created automatically. Tables are created automatically.
@ -115,6 +115,10 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False):
`verbose` `verbose`
If set to True, status messages will be printed to stdout. If set to True, status messages will be printed to stdout.
`safe`
If set to False, load can be faster, but can corrupt the database if
it crashes or is interrupted.
""" """
# First take care of verbosity # First take care of verbosity
@ -128,6 +132,10 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False):
table_objs = [metadata.tables[name] for name in table_names] table_objs = [metadata.tables[name] for name in table_names]
table_objs = sqlalchemy.sql.util.sort_tables(table_objs) table_objs = sqlalchemy.sql.util.sort_tables(table_objs)
# SQLite speed tweaks
if not safe and session.connection().dialect.name == 'sqlite':
session.connection().execute("PRAGMA synchronous=OFF")
session.connection().execute("PRAGMA journal_mode=OFF")
# Drop all tables if requested # Drop all tables if requested
if drop_tables: if drop_tables:
@ -251,6 +259,10 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False):
print_done() print_done()
# SQLite check
if session.connection().dialect.name == 'sqlite':
session.connection().execute("PRAGMA integrity_check")
def dump(session, tables=[], directory=None, verbose=False): def dump(session, tables=[], directory=None, verbose=False):