* fix hyperdandik tokenization a bit.

- include the last token out of the loop if not empty - treat punctuation as whitespace
2005-12-12 23:40:59 +00:00
parent cb7893610e
commit 81d11c8172
1 changed files with 4 additions and 3 deletions
@@ -21,13 +21,14 @@ def tokenize(lang, str):
    tokens = []
    token = unicode()
    for x in str:
-        if x in string.whitespace:
+        if x in string.whitespace or x in string.punctuation:
            if len(token) > 0:
                tokens.append(token)
                token = unicode()
-        elif x in string.punctuation:
-            pass # eat punctuation
        else:
            token += x
+            
+    if token:
+        tokens.append(x)

    return tokens