This commit is contained in:
Eray Özkural
2005-12-12 23:44:53 +00:00
parent 3c3c88e207
commit 606464ec98
+1 -1
View File
@@ -29,6 +29,6 @@ def tokenize(lang, str):
token += x token += x
if token: if token:
tokens.append(x) tokens.append(token)
return tokens return tokens