Browse Source

added basic parser

Inderjit Gill 4 months ago
parent
commit
b8c6d2bad5
5 changed files with 499 additions and 14 deletions
  1. 25
    14
      Cargo.lock
  2. 33
    0
      samplescene.rts
  3. 9
    0
      src/error.rs
  4. 2
    0
      src/main.rs
  5. 430
    0
      src/parser.rs

+ 25
- 14
Cargo.lock View File

@@ -59,7 +59,7 @@ name = "backtrace-sys"
59 59
 version = "0.1.24"
60 60
 source = "registry+https://github.com/rust-lang/crates.io-index"
61 61
 dependencies = [
62
- "cc 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)",
62
+ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)",
63 63
  "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
64 64
 ]
65 65
 
@@ -75,7 +75,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
75 75
 
76 76
 [[package]]
77 77
 name = "cc"
78
-version = "1.0.19"
78
+version = "1.0.25"
79 79
 source = "registry+https://github.com/rust-lang/crates.io-index"
80 80
 
81 81
 [[package]]
@@ -179,22 +179,22 @@ dependencies = [
179 179
 
180 180
 [[package]]
181 181
 name = "failure"
182
-version = "0.1.2"
182
+version = "0.1.3"
183 183
 source = "registry+https://github.com/rust-lang/crates.io-index"
184 184
 dependencies = [
185 185
  "backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
186
- "failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
186
+ "failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
187 187
 ]
188 188
 
189 189
 [[package]]
190 190
 name = "failure_derive"
191
-version = "0.1.2"
191
+version = "0.1.3"
192 192
 source = "registry+https://github.com/rust-lang/crates.io-index"
193 193
 dependencies = [
194 194
  "proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
195 195
  "quote 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
196
- "syn 0.14.8 (registry+https://github.com/rust-lang/crates.io-index)",
197
- "synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
196
+ "syn 0.15.22 (registry+https://github.com/rust-lang/crates.io-index)",
197
+ "synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
198 198
 ]
199 199
 
200 200
 [[package]]
@@ -430,7 +430,7 @@ dependencies = [
430 430
  "cgmath 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)",
431 431
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
432 432
  "env_logger 0.5.12 (registry+https://github.com/rust-lang/crates.io-index)",
433
- "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
433
+ "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
434 434
  "image 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)",
435 435
  "log 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
436 436
  "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -521,13 +521,23 @@ dependencies = [
521 521
 ]
522 522
 
523 523
 [[package]]
524
+name = "syn"
525
+version = "0.15.22"
526
+source = "registry+https://github.com/rust-lang/crates.io-index"
527
+dependencies = [
528
+ "proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
529
+ "quote 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
530
+ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
531
+]
532
+
533
+[[package]]
524 534
 name = "synstructure"
525
-version = "0.9.0"
535
+version = "0.10.1"
526 536
 source = "registry+https://github.com/rust-lang/crates.io-index"
527 537
 dependencies = [
528 538
  "proc-macro2 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
529 539
  "quote 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
530
- "syn 0.14.8 (registry+https://github.com/rust-lang/crates.io-index)",
540
+ "syn 0.15.22 (registry+https://github.com/rust-lang/crates.io-index)",
531 541
  "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
532 542
 ]
533 543
 
@@ -633,7 +643,7 @@ dependencies = [
633 643
 "checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0"
634 644
 "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12"
635 645
 "checksum byteorder 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8389c509ec62b9fe8eca58c502a0acaf017737355615243496cde4994f8fa4f9"
636
-"checksum cc 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)" = "850d5f0c639adf4715b8cdf0272db2a4f1da26135b180e1f94bac592a8fc8e6d"
646
+"checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16"
637 647
 "checksum cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4e7bb64a8ebb0d856483e1e682ea3422f883c5f5615a90d51a2c82fe87fdd3"
638 648
 "checksum cgmath 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)" = "64a4b57c8f4e3a2e9ac07e0f6abc9c24b6fc9e1b54c3478cfb598f3d0023e51c"
639 649
 "checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e"
@@ -645,8 +655,8 @@ dependencies = [
645 655
 "checksum deflate 0.7.18 (registry+https://github.com/rust-lang/crates.io-index)" = "32c8120d981901a9970a3a1c97cf8b630e0fa8c3ca31e75b6fd6fd5f9f427b31"
646 656
 "checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0"
647 657
 "checksum env_logger 0.5.12 (registry+https://github.com/rust-lang/crates.io-index)" = "f4d7e69c283751083d53d01eac767407343b8b69c4bd70058e08adc2637cb257"
648
-"checksum failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7efb22686e4a466b1ec1a15c2898f91fa9cb340452496dca654032de20ff95b9"
649
-"checksum failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "946d0e98a50d9831f5d589038d2ca7f8f455b1c21028c0db0e84116a12696426"
658
+"checksum failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6dd377bcc1b1b7ce911967e3ec24fa19c3224394ec05b54aa7b083d498341ac7"
659
+"checksum failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "64c2d913fe8ed3b6c6518eedf4538255b989945c14c2a7d5cbff62a5e2120596"
650 660
 "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
651 661
 "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
652 662
 "checksum gif 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff3414b424657317e708489d2857d9575f4403698428b040b609b9d1c1a84a2c"
@@ -686,7 +696,8 @@ dependencies = [
686 696
 "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
687 697
 "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
688 698
 "checksum syn 0.14.8 (registry+https://github.com/rust-lang/crates.io-index)" = "b7bfcbb0c068d0f642a0ffbd5c604965a360a61f99e8add013cef23a838614f3"
689
-"checksum synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb9b7550d063ea184027c9b8c20ac167cd36d3e06b3a40bceb9d746dc1a7b7"
699
+"checksum syn 0.15.22 (registry+https://github.com/rust-lang/crates.io-index)" = "ae8b29eb5210bc5cf63ed6149cbf9adfc82ac0be023d8735c176ee74a2db4da7"
700
+"checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015"
690 701
 "checksum termcolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "722426c4a0539da2c4ffd9b419d90ad540b4cff4a053be9069c908d4d07e2836"
691 702
 "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
692 703
 "checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6"

+ 33
- 0
samplescene.rts View File

@@ -0,0 +1,33 @@
1
+;; rts = ray trace scene
2
+
3
+(sphere center: [0.0 0.0 -1.0]
4
+        radius: 0.5
5
+        material: (lambertian albedo: [0.8 0.3 0.3]))
6
+
7
+(sphere center: [0.0 -100.5 -1.0]
8
+        radius: 100
9
+        material: (lambertian albedo: [0.8 0.8 0.0]))
10
+
11
+(sphere center: [1.0 0.0 -1.0]
12
+        radius: 0.5
13
+        material: (metal albedo: [0.8 0.6 0.2]
14
+                         fuzz: 1.0))
15
+
16
+(sphere center: [-1.0 0.0 -1.0]
17
+        radius: 0.5
18
+        material: (metal albedo: [0.8 0.8 0.8]
19
+                         fuzz: 0.3))
20
+
21
+;; default picture parameters that can be overridden on the command line
22
+(picture width: 100
23
+         height: 100
24
+         samples: 1
25
+         output: "foo.png")
26
+
27
+(camera look-from: [0.0, 0.0, 0.0]
28
+        look-at: [0.0, 0.0, -1.0]
29
+        vup: [0.0, 1.0, 0.0]
30
+        vfov: 90
31
+        aspect: 1                       ; leave out == calculate from picture parameters
32
+        aperture: 0                     ; leave out == calculate from look-from, look-at
33
+        focus-dist: 1)

+ 9
- 0
src/error.rs View File

@@ -16,6 +16,15 @@ pub enum RayTracerError {
16 16
     CommandLineParsingError,
17 17
     // #[fail(display = "Image size mismatch")] SizeMismatch,
18 18
 
19
+    #[fail(display = "Parser invalid char")]
20
+    ParserInvalidChar(char),
21
+    #[fail(display = "Parser invalid literal")]
22
+    ParserInvalidLiteral,
23
+    #[fail(display = "Parser unable to parse float")]
24
+    ParserUnableToParseFloat(String),
25
+    #[fail(display = "Parser handled token")]
26
+    ParserHandledToken,
27
+
19 28
     // #[fail(display = "No first image")] NoFirstImage,
20 29
 }
21 30
 

+ 2
- 0
src/main.rs View File

@@ -22,6 +22,7 @@ mod rng;
22 22
 mod scene;
23 23
 mod shape;
24 24
 mod types;
25
+mod parser;
25 26
 
26 27
 use cgmath::prelude::*;
27 28
 use clap::{App, Arg};
@@ -35,6 +36,7 @@ use rng::ThreadLocalRng;
35 36
 use scene::*;
36 37
 use shape::*;
37 38
 use types::*;
39
+use parser::*;
38 40
 
39 41
 fn main() -> Result<()> {
40 42
     env_logger::init_from_env("RAYTRACER_LOG");

+ 430
- 0
src/parser.rs View File

@@ -0,0 +1,430 @@
1
+use error::{Result, RayTracerError};
2
+
3
+#[derive(Debug, PartialEq)]
4
+pub enum Node {
5
+    List(Vec<Node>),
6
+    Vector(Vec<Node>),
7
+    Float(f32),
8
+    Name(String),
9
+    Label(String),
10
+    String(String),
11
+    Comment(String),
12
+    End,
13
+}
14
+
15
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
16
+enum Token<'a> {
17
+    Colon,
18
+    Comment(&'a str),
19
+    DoubleQuote,
20
+    Name(&'a str),
21
+    Number(&'a str),
22
+    ParenEnd,
23
+    ParenStart,
24
+    SquareBracketEnd,
25
+    SquareBracketStart,
26
+    Whitespace(&'a str),
27
+    End,
28
+}
29
+
30
+struct NodeAndRemainder<'a> {
31
+    node: Node,
32
+    tokens: &'a [Token<'a>],
33
+}
34
+
35
+pub fn parse(s: &str) -> Result<Vec<Node>> {
36
+    let t = tokenize(s)?;
37
+
38
+    let mut tokens = t.as_slice();
39
+    let mut res = Vec::new();
40
+
41
+    while tokens.len() > 0 {
42
+        match eat_token(tokens) {
43
+            Ok(nar) => {
44
+                match nar.node {
45
+                    Node::End => (),
46
+                    _ => res.push(nar.node),
47
+                }
48
+                tokens = nar.tokens;
49
+            },
50
+            Err(e) => return Err(e)
51
+        }
52
+    }
53
+
54
+    Ok(res)
55
+}
56
+
57
+fn tokenize(s: &str) -> Result<Vec<Token>> {
58
+    let mut lex = Lexer::new(s);
59
+    let mut res = Vec::new();
60
+
61
+    loop {
62
+        match lex.eat_token()? {
63
+            Token::End => break,
64
+            tok => res.push(tok),
65
+        }
66
+    }
67
+
68
+    Ok(res)
69
+}
70
+
71
+struct Lexer<'a> {
72
+    input: &'a str,
73
+    cur_pos: u32,
74
+}
75
+
76
+impl<'a> Lexer<'a> {
77
+    pub fn new(input: &str) -> Lexer {
78
+        Lexer{
79
+            input: input,
80
+            cur_pos: 0,
81
+        }
82
+    }
83
+
84
+    pub fn eat_token(&mut self) -> Result<Token<'a>> {
85
+        let mut chars = self.input.char_indices();
86
+
87
+        while let Some((ind, ch)) = chars.next() {
88
+            let res = match ch {
89
+                '(' => Ok((Token::ParenStart, 1)),
90
+                ')' => Ok((Token::ParenEnd, 1)),
91
+                '[' => Ok((Token::SquareBracketStart, 1)),
92
+                ']' => Ok((Token::SquareBracketEnd, 1)),
93
+                ':' => Ok((Token::Colon, 1)),
94
+                '"' => Ok((Token::DoubleQuote, 1)),
95
+                ';' => eat_comment(&self.input[ind..]),
96
+                '-' | '0' ... '9' => eat_number(&self.input[ind..]),
97
+                ch if ch.is_whitespace() => eat_whitespace(&self.input[ind..]),
98
+                _ if is_name(ch) => eat_name(&self.input[ind..]),
99
+                ch => Err(RayTracerError::ParserInvalidChar(ch))
100
+            };
101
+
102
+            let (tok, size) = match res {
103
+                Ok(v) => v,
104
+                Err(kind) => return Err(kind)
105
+            };
106
+
107
+            self.cur_pos += size as u32;
108
+            self.input = &self.input[ind + size..];
109
+
110
+            return Ok(tok);
111
+        }
112
+
113
+        self.input = &self.input[..0];
114
+        Ok(Token::End)
115
+    }
116
+}
117
+
118
+fn is_name(ch: char) -> bool {
119
+    ch.is_alphanumeric() || is_symbol(ch)
120
+}
121
+
122
+fn is_symbol(ch: char) -> bool {
123
+    match ch {
124
+        '+' | '-' | '*' | '/' | '=' | '!' | '@' |
125
+        '#' | '$' | '%' | '^' | '&' | '<' | '>' |
126
+        '?' | '.' => true,
127
+        _ => false
128
+    }
129
+}
130
+
131
+fn eat_name(input: &str) -> Result<(Token, usize)> {
132
+    for (ind, ch) in input.char_indices() {
133
+        if !is_name(ch) {
134
+            return Ok((Token::Name(&input[..ind]), ind));
135
+        }
136
+    }
137
+
138
+    Ok((Token::Name(input), input.len()))
139
+}
140
+
141
+fn eat_whitespace(input: &str) -> Result<(Token, usize)> {
142
+    for (ind, ch) in input.char_indices() {
143
+        if !ch.is_whitespace() {
144
+            return Ok((Token::Whitespace(&input[..ind]), ind));
145
+        }
146
+    }
147
+
148
+    Ok((Token::Whitespace(input), input.len()))
149
+}
150
+
151
+fn eat_number(input: &str) -> Result<(Token, usize)> {
152
+    let mut digits = false;
153
+    let mut dot = false;
154
+    let mut size = input.len();
155
+
156
+    let (prefix_offset, rest) = if input.starts_with('-') {
157
+        match input[1..].chars().next() {
158
+            Some(ch) if ch.is_digit(10) => (1, &input[1..]),
159
+            // Actually a name beginning with '-' rather a number
160
+            _ => return eat_name(input)
161
+        }
162
+    } else {
163
+        (0, input)
164
+    };
165
+
166
+    for (ind, ch) in rest.char_indices() {
167
+        match ch {
168
+            '.' => {
169
+                if dot {
170
+                    return Err(RayTracerError::ParserInvalidLiteral);
171
+                }
172
+                dot = true;
173
+            }
174
+            _ if ch.is_digit(10) => {
175
+                digits = true;
176
+            }
177
+            _ => {
178
+                size = prefix_offset + ind;
179
+                break;
180
+            }
181
+        }
182
+    }
183
+
184
+    if !digits {
185
+        Err(RayTracerError::ParserInvalidLiteral)
186
+    } else {
187
+        Ok((Token::Number(&input[..size]), size))
188
+    }
189
+}
190
+
191
+fn eat_comment(input: &str) -> Result<(Token, usize)> {
192
+    let rest = &input[1..];     // remove the first character (;)
193
+    let mut size = rest.len();
194
+
195
+    for (ind, ch) in rest.char_indices() {
196
+        match ch {
197
+            '\n' =>{
198
+                size = ind;
199
+                break;
200
+            }
201
+            _ => {
202
+                continue;
203
+            }
204
+        }
205
+    }
206
+    Ok((Token::Comment(&rest[..size]), size + 1))
207
+}
208
+
209
+fn eat_quote<'a>(t: &'a [Token<'a>]) -> Result<NodeAndRemainder<'a>> {
210
+    let mut tokens = t;
211
+    let mut content: String = "".to_string();
212
+
213
+    loop {
214
+        match tokens[0] {
215
+            Token::DoubleQuote => return Ok(NodeAndRemainder {
216
+                node: Node::String(content),
217
+                tokens: &tokens[1..],
218
+            }),
219
+            _ => {
220
+                match eat_token(tokens) {
221
+                    Ok(nar) => {
222
+                        match nar.node {
223
+                            Node::Name(txt) => content = txt,
224
+                            _ => ()
225
+                        }
226
+                        tokens = nar.tokens;
227
+                    },
228
+                    Err(e) => return Err(e)
229
+                }
230
+            }
231
+        }
232
+    }
233
+}
234
+
235
+// At the first token after a Token::ParenStart
236
+//
237
+fn eat_list<'a>(t: &'a [Token<'a>]) -> Result<NodeAndRemainder<'a>> {
238
+    let mut tokens = t;
239
+    let mut res: Vec<Node> = Vec::new();
240
+
241
+    loop {
242
+        match tokens[0] {
243
+            Token::ParenEnd => return Ok(NodeAndRemainder {
244
+                node: Node::List(res),
245
+                tokens: &tokens[1..],
246
+            }),
247
+            _ => {
248
+                match eat_token(tokens) {
249
+                    Ok(nar) => {
250
+                        res.push(nar.node);
251
+                        tokens = nar.tokens;
252
+                    },
253
+                    Err(e) => return Err(e)
254
+                }
255
+            }
256
+        }
257
+    }
258
+}
259
+
260
+fn eat_vector<'a>(t: &'a [Token<'a>]) -> Result<NodeAndRemainder<'a>> {
261
+    let mut tokens = t;
262
+    let mut res: Vec<Node> = Vec::new();
263
+
264
+    loop {
265
+        match tokens[0] {
266
+            Token::SquareBracketEnd => return Ok(NodeAndRemainder {
267
+                node: Node::Vector(res),
268
+                tokens: &tokens[1..],
269
+            }),
270
+            _ => {
271
+                match eat_token(tokens) {
272
+                    Ok(nar) => {
273
+                        res.push(nar.node);
274
+                        tokens = nar.tokens;
275
+                    },
276
+                    Err(e) => return Err(e)
277
+                }
278
+            }
279
+        }
280
+    }
281
+}
282
+
283
+fn eat_token<'a>(t: &'a[Token<'a>]) -> Result<NodeAndRemainder<'a>> {
284
+    let mut tokens = t;
285
+
286
+    loop {
287
+        if tokens.is_empty() {
288
+            return Ok(NodeAndRemainder {
289
+                        node: Node::End,
290
+                        tokens,
291
+                    })
292
+        }
293
+
294
+        match tokens[0] {
295
+            // skip whitespace tokens
296
+            Token::Whitespace(_) => tokens = &tokens[1..],
297
+            Token::Comment(_) => tokens = &tokens[1..],
298
+            Token::DoubleQuote => return eat_quote(&tokens[1..]),
299
+            Token::Name(txt) =>
300
+                if tokens.len() > 1 && tokens[1] == Token::Colon {
301
+                    return Ok(NodeAndRemainder {
302
+                        node: Node::Label(txt.to_string()),
303
+                        tokens: &tokens[2..],
304
+                    })
305
+                } else {
306
+                    return Ok(NodeAndRemainder {
307
+                        node: Node::Name(txt.to_string()),
308
+                        tokens: &tokens[1..],
309
+                    })
310
+                },
311
+            Token::Number(txt) => {
312
+                return match txt.parse::<f32>() {
313
+                    Ok(f) => Ok(NodeAndRemainder {
314
+                        node: Node::Float(f),
315
+                        tokens: &tokens[1..],
316
+                    }),
317
+                    Err(_) => Err(RayTracerError::ParserUnableToParseFloat(txt.to_string()))
318
+                }
319
+            },
320
+            Token::ParenStart => return eat_list(&tokens[1..]),
321
+            Token::SquareBracketStart => return eat_vector(&tokens[1..]),
322
+            _ => return Err(RayTracerError::ParserHandledToken),
323
+        }
324
+    }
325
+}
326
+
327
+#[cfg(test)]
328
+mod tests {
329
+    use super::*;
330
+
331
+    #[test]
332
+    fn test_lexer() {
333
+        assert_eq!(tokenize("()").unwrap(),
334
+                   [Token::ParenStart,
335
+                    Token::ParenEnd]);
336
+
337
+        assert_eq!(tokenize("( )").unwrap(),
338
+                   [Token::ParenStart,
339
+                    Token::Whitespace(" "),
340
+                    Token::ParenEnd]);
341
+
342
+        assert_eq!(tokenize("[]").unwrap(),
343
+                   [Token::SquareBracketStart,
344
+                    Token::SquareBracketEnd]);
345
+
346
+        assert_eq!(tokenize("5").unwrap(),
347
+                   [Token::Number("5")]);
348
+        assert_eq!(tokenize("-3").unwrap(),
349
+                   [Token::Number("-3")]);
350
+        assert_eq!(tokenize("3.14").unwrap(),
351
+                   [Token::Number("3.14")]);
352
+        assert_eq!(tokenize("-0.34").unwrap(),
353
+                   [Token::Number("-0.34")]);
354
+
355
+        assert_eq!(tokenize("1 foo 3").unwrap(),
356
+                   [Token::Number("1"),
357
+                    Token::Whitespace(" "),
358
+                    Token::Name("foo"),
359
+                    Token::Whitespace(" "),
360
+                    Token::Number("3")]);
361
+
362
+        assert_eq!(tokenize("\"hello/world.\"").unwrap(),
363
+                   [Token::DoubleQuote,
364
+                    Token::Name("hello/world."),
365
+                    Token::DoubleQuote]);
366
+
367
+        assert_eq!(tokenize("hello\nworld").unwrap(),
368
+                   [Token::Name("hello"),
369
+                    Token::Whitespace("\n"),
370
+                    Token::Name("world")]);
371
+
372
+        assert_eq!(tokenize("hello ; some comment").unwrap(),
373
+                   [Token::Name("hello"),
374
+                    Token::Whitespace(" "),
375
+                    Token::Comment(" some comment")]);
376
+
377
+        assert_eq!(tokenize("hello ; some comment\n(valid)").unwrap(),
378
+                   [Token::Name("hello"),
379
+                    Token::Whitespace(" "),
380
+                    Token::Comment(" some comment"),
381
+                    Token::Whitespace("\n"),
382
+                    Token::ParenStart,
383
+                    Token::Name("valid"),
384
+                    Token::ParenEnd]);
385
+    }
386
+
387
+    fn ast(s: &str) -> Vec<Node> {
388
+        parse(s).unwrap()
389
+    }
390
+
391
+    #[test]
392
+    fn test_parser() {
393
+        assert_eq!(ast("hello"),
394
+                   [Node::Name("hello".to_string())]);
395
+        assert_eq!(ast("hello world"),
396
+                   [Node::Name("hello".to_string()),
397
+                    Node::Name("world".to_string())]);
398
+        assert_eq!(ast("hello: world"),
399
+                   [Node::Label("hello".to_string()),
400
+                    Node::Name("world".to_string())]);
401
+        assert_eq!(ast("42 102"),
402
+                   [Node::Float(42.0),
403
+                    Node::Float(102.0)]);
404
+
405
+        assert_eq!(ast("\"path/to/texture.png\""),
406
+                   [Node::String("path/to/texture.png".to_string())]);
407
+
408
+        assert_eq!(ast("hello world ; some comment"),
409
+                   [Node::Name("hello".to_string()),
410
+                    Node::Name("world".to_string()),]);
411
+
412
+        assert_eq!(ast("(hello world)"),
413
+                   [Node::List(vec![Node::Name("hello".to_string()),
414
+                                    Node::Name("world".to_string())])]);
415
+
416
+        assert_eq!(ast("(hello world (1 2 3))"),
417
+                   [Node::List(vec![Node::Name("hello".to_string()),
418
+                                    Node::Name("world".to_string()),
419
+                                    Node::List(vec![Node::Float(1.0),
420
+                                                    Node::Float(2.0),
421
+                                                    Node::Float(3.0)])])]);
422
+
423
+        assert_eq!(ast("(hello world [1 2 3])"),
424
+                   [Node::List(vec![Node::Name("hello".to_string()),
425
+                                    Node::Name("world".to_string()),
426
+                                    Node::Vector(vec![Node::Float(1.0),
427
+                                                      Node::Float(2.0),
428
+                                                      Node::Float(3.0)])])]);
429
+    }
430
+}