Adding in some lexer testing.

test.py

@@ -1,10 +0,0 @@
-
-from hy.lex.tokenize import tokenize
-
-print tokenize("""
-(+ 2 (+ 1 1) (- 1 1))
-""")
-
-print tokenize("""
-(print "Hello, \\n World")
-""")

tests/lexer/test_basic_lexing.py

@@ -0,0 +1,20 @@
+from hy.lex.tokenize import tokenize
+
+
+def test_simple_tokenize():
+    assert [["+", "1", "1"]] == tokenize("(+ 1 1)")
+
+
+def test_double_tokenize():
+    assert [
+        ["+", "1", "2"],
+        ["-", "1", "1"]
+    ] == tokenize("(+ 1 2) (- 1 1)")
+
+
+def test_simple_recurse():
+    assert [
+        '+', '1', [
+            '+', '1', '1'
+        ]
+    ] == tokenize("(+ 1 (+ 1 1))")