Skip to content
Snippets Groups Projects
Commit 9d419f48 authored by David Baker's avatar David Baker
Browse files

Make the port script drop NUL values in all tables

Postgres doesn't support NULs in strings so it makes the script
throw an exception and stop if any values contain \0. Drop them
with appropriate warning.
parent c31a7c3f
No related branches found
No related tags found
No related merge requests found
......@@ -320,7 +320,7 @@ class Porter(object):
backward_chunk = min(row[0] for row in brows) - 1
rows = frows + brows
self._convert_rows(table, headers, rows)
rows = self._convert_rows(table, headers, rows)
def insert(txn):
self.postgres_store.insert_many_txn(
......@@ -556,17 +556,29 @@ class Porter(object):
i for i, h in enumerate(headers) if h in bool_col_names
]
class BadValueException(Exception):
pass
def conv(j, col):
if j in bool_cols:
return bool(col)
elif isinstance(col, basestring) and "\0" in col:
logger.warn("DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col)
raise BadValueException();
return col
outrows = []
for i, row in enumerate(rows):
rows[i] = tuple(
conv(j, col)
for j, col in enumerate(row)
if j > 0
)
try:
outrows.append(tuple(
conv(j, col)
for j, col in enumerate(row)
if j > 0
))
except BadValueException:
pass
return outrows
@defer.inlineCallbacks
def _setup_sent_transactions(self):
......@@ -594,7 +606,7 @@ class Porter(object):
"select", r,
)
self._convert_rows("sent_transactions", headers, rows)
rows = self._convert_rows("sent_transactions", headers, rows)
inserted_rows = len(rows)
if inserted_rows:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment