| 1 | # -*- coding: iso-8859-1 -*- |
|---|
| 2 | """A lexical analyzer class for simple shell-like syntaxes.""" |
|---|
| 3 | |
|---|
| 4 | # Module and documentation by Eric S. Raymond, 21 Dec 1998 |
|---|
| 5 | # Input stacking and error message cleanup added by ESR, March 2000 |
|---|
| 6 | # push_source() and pop_source() made explicit by ESR, January 2001. |
|---|
| 7 | # Posix compliance, split(), string arguments, and |
|---|
| 8 | # iterator interface by Gustavo Niemeyer, April 2003. |
|---|
| 9 | |
|---|
| 10 | import os.path |
|---|
| 11 | import sys |
|---|
| 12 | from collections import deque |
|---|
| 13 | |
|---|
| 14 | try: |
|---|
| 15 | from cStringIO import StringIO |
|---|
| 16 | except ImportError: |
|---|
| 17 | from StringIO import StringIO |
|---|
| 18 | |
|---|
| 19 | __all__ = ["shlex", "split"] |
|---|
| 20 | |
|---|
| 21 | class shlex: |
|---|
| 22 | "A lexical analyzer class for simple shell-like syntaxes." |
|---|
| 23 | def __init__(self, instream=None, infile=None, posix=False): |
|---|
| 24 | if isinstance(instream, basestring): |
|---|
| 25 | instream = StringIO(instream) |
|---|
| 26 | if instream is not None: |
|---|
| 27 | self.instream = instream |
|---|
| 28 | self.infile = infile |
|---|
| 29 | else: |
|---|
| 30 | self.instream = sys.stdin |
|---|
| 31 | self.infile = None |
|---|
| 32 | self.posix = posix |
|---|
| 33 | if posix: |
|---|
| 34 | self.eof = None |
|---|
| 35 | else: |
|---|
| 36 | self.eof = '' |
|---|
| 37 | self.commenters = '#' |
|---|
| 38 | self.wordchars = ('abcdfeghijklmnopqrstuvwxyz' |
|---|
| 39 | 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_') |
|---|
| 40 | if self.posix: |
|---|
| 41 | self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ' |
|---|
| 42 | 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ') |
|---|
| 43 | self.whitespace = ' \t\r\n' |
|---|
| 44 | self.whitespace_split = False |
|---|
| 45 | self.quotes = '\'"' |
|---|
| 46 | self.escape = '\\' |
|---|
| 47 | self.escapedquotes = '"' |
|---|
| 48 | self.state = ' ' |
|---|
| 49 | self.pushback = deque() |
|---|
| 50 | self.lineno = 1 |
|---|
| 51 | self.debug = 0 |
|---|
| 52 | self.token = '' |
|---|
| 53 | self.filestack = deque() |
|---|
| 54 | self.source = None |
|---|
| 55 | if self.debug: |
|---|
| 56 | print 'shlex: reading from %s, line %d' \ |
|---|
| 57 | % (self.instream, self.lineno) |
|---|
| 58 | |
|---|
| 59 | def push_token(self, tok): |
|---|
| 60 | "Push a token onto the stack popped by the get_token method" |
|---|
| 61 | if self.debug >= 1: |
|---|
| 62 | print "shlex: pushing token " + repr(tok) |
|---|
| 63 | self.pushback.appendleft(tok) |
|---|
| 64 | |
|---|
| 65 | def push_source(self, newstream, newfile=None): |
|---|
| 66 | "Push an input source onto the lexer's input source stack." |
|---|
| 67 | if isinstance(newstream, basestring): |
|---|
| 68 | newstream = StringIO(newstream) |
|---|
| 69 | self.filestack.appendleft((self.infile, self.instream, self.lineno)) |
|---|
| 70 | self.infile = newfile |
|---|
| 71 | self.instream = newstream |
|---|
| 72 | self.lineno = 1 |
|---|
| 73 | if self.debug: |
|---|
| 74 | if newfile is not None: |
|---|
| 75 | print 'shlex: pushing to file %s' % (self.infile,) |
|---|
| 76 | else: |
|---|
| 77 | print 'shlex: pushing to stream %s' % (self.instream,) |
|---|
| 78 | |
|---|
| 79 | def pop_source(self): |
|---|
| 80 | "Pop the input source stack." |
|---|
| 81 | self.instream.close() |
|---|
| 82 | (self.infile, self.instream, self.lineno) = self.filestack.popleft() |
|---|
| 83 | if self.debug: |
|---|
| 84 | print 'shlex: popping to %s, line %d' \ |
|---|
| 85 | % (self.instream, self.lineno) |
|---|
| 86 | self.state = ' ' |
|---|
| 87 | |
|---|
| 88 | def get_token(self): |
|---|
| 89 | "Get a token from the input stream (or from stack if it's nonempty)" |
|---|
| 90 | if self.pushback: |
|---|
| 91 | tok = self.pushback.popleft() |
|---|
| 92 | if self.debug >= 1: |
|---|
| 93 | print "shlex: popping token " + repr(tok) |
|---|
| 94 | return tok |
|---|
| 95 | # No pushback. Get a token. |
|---|
| 96 | raw = self.read_token() |
|---|
| 97 | # Handle inclusions |
|---|
| 98 | if self.source is not None: |
|---|
| 99 | while raw == self.source: |
|---|
| 100 | spec = self.sourcehook(self.read_token()) |
|---|
| 101 | if spec: |
|---|
| 102 | (newfile, newstream) = spec |
|---|
| 103 | self.push_source(newstream, newfile) |
|---|
| 104 | raw = self.get_token() |
|---|
| 105 | # Maybe we got EOF instead? |
|---|
| 106 | while raw == self.eof: |
|---|
| 107 | if not self.filestack: |
|---|
| 108 | return self.eof |
|---|
| 109 | else: |
|---|
| 110 | self.pop_source() |
|---|
| 111 | raw = self.get_token() |
|---|
| 112 | # Neither inclusion nor EOF |
|---|
| 113 | if self.debug >= 1: |
|---|
| 114 | if raw != self.eof: |
|---|
| 115 | print "shlex: token=" + repr(raw) |
|---|
| 116 | else: |
|---|
| 117 | print "shlex: token=EOF" |
|---|
| 118 | return raw |
|---|
| 119 | |
|---|
| 120 | def read_token(self): |
|---|
| 121 | quoted = False |
|---|
| 122 | escapedstate = ' ' |
|---|
| 123 | while True: |
|---|
| 124 | nextchar = self.instream.read(1) |
|---|
| 125 | if nextchar == '\n': |
|---|
| 126 | self.lineno = self.lineno + 1 |
|---|
| 127 | if self.debug >= 3: |
|---|
| 128 | print "shlex: in state", repr(self.state), \ |
|---|
| 129 | "I see character:", repr(nextchar) |
|---|
| 130 | if self.state is None: |
|---|
| 131 | self.token = '' # past end of file |
|---|
| 132 | break |
|---|
| 133 | elif self.state == ' ': |
|---|
| 134 | if not nextchar: |
|---|
| 135 | self.state = None # end of file |
|---|
| 136 | break |
|---|
| 137 | elif nextchar in self.whitespace: |
|---|
| 138 | if self.debug >= 2: |
|---|
| 139 | print "shlex: I see whitespace in whitespace state" |
|---|
| 140 | if self.token or (self.posix and quoted): |
|---|
| 141 | break # emit current token |
|---|
| 142 | else: |
|---|
| 143 | continue |
|---|
| 144 | elif nextchar in self.commenters: |
|---|
| 145 | self.instream.readline() |
|---|
| 146 | self.lineno = self.lineno + 1 |
|---|
| 147 | elif self.posix and nextchar in self.escape: |
|---|
| 148 | escapedstate = 'a' |
|---|
| 149 | self.state = nextchar |
|---|
| 150 | elif nextchar in self.wordchars: |
|---|
| 151 | self.token = nextchar |
|---|
| 152 | self.state = 'a' |
|---|
| 153 | elif nextchar in self.quotes: |
|---|
| 154 | if not self.posix: |
|---|
| 155 | self.token = nextchar |
|---|
| 156 | self.state = nextchar |
|---|
| 157 | elif self.whitespace_split: |
|---|
| 158 | self.token = nextchar |
|---|
| 159 | self.state = 'a' |
|---|
| 160 | else: |
|---|
| 161 | self.token = nextchar |
|---|
| 162 | if self.token or (self.posix and quoted): |
|---|
| 163 | break # emit current token |
|---|
| 164 | else: |
|---|
| 165 | continue |
|---|
| 166 | elif self.state in self.quotes: |
|---|
| 167 | quoted = True |
|---|
| 168 | if not nextchar: # end of file |
|---|
| 169 | if self.debug >= 2: |
|---|
| 170 | print "shlex: I see EOF in quotes state" |
|---|
| 171 | # XXX what error should be raised here? |
|---|
| 172 | raise ValueError, "No closing quotation" |
|---|
| 173 | if nextchar == self.state: |
|---|
| 174 | if not self.posix: |
|---|
| 175 | self.token = self.token + nextchar |
|---|
| 176 | self.state = ' ' |
|---|
| 177 | break |
|---|
| 178 | else: |
|---|
| 179 | self.state = 'a' |
|---|
| 180 | elif self.posix and nextchar in self.escape and \ |
|---|
| 181 | self.state in self.escapedquotes: |
|---|
| 182 | escapedstate = self.state |
|---|
| 183 | self.state = nextchar |
|---|
| 184 | else: |
|---|
| 185 | self.token = self.token + nextchar |
|---|
| 186 | elif self.state in self.escape: |
|---|
| 187 | if not nextchar: # end of file |
|---|
| 188 | if self.debug >= 2: |
|---|
| 189 | print "shlex: I see EOF in escape state" |
|---|
| 190 | # XXX what error should be raised here? |
|---|
| 191 | raise ValueError, "No escaped character" |
|---|
| 192 | # In posix shells, only the quote itself or the escape |
|---|
| 193 | # character may be escaped within quotes. |
|---|
| 194 | if escapedstate in self.quotes and \ |
|---|
| 195 | nextchar != self.state and nextchar != escapedstate: |
|---|
| 196 | self.token = self.token + self.state |
|---|
| 197 | self.token = self.token + nextchar |
|---|
| 198 | self.state = escapedstate |
|---|
| 199 | elif self.state == 'a': |
|---|
| 200 | if not nextchar: |
|---|
| 201 | self.state = None # end of file |
|---|
| 202 | break |
|---|
| 203 | elif nextchar in self.whitespace: |
|---|
| 204 | if self.debug >= 2: |
|---|
| 205 | print "shlex: I see whitespace in word state" |
|---|
| 206 | self.state = ' ' |
|---|
| 207 | if self.token or (self.posix and quoted): |
|---|
| 208 | break # emit current token |
|---|
| 209 | else: |
|---|
| 210 | continue |
|---|
| 211 | elif nextchar in self.commenters: |
|---|
| 212 | self.instream.readline() |
|---|
| 213 | self.lineno = self.lineno + 1 |
|---|
| 214 | if self.posix: |
|---|
| 215 | self.state = ' ' |
|---|
| 216 | if self.token or (self.posix and quoted): |
|---|
| 217 | break # emit current token |
|---|
| 218 | else: |
|---|
| 219 | continue |
|---|
| 220 | elif self.posix and nextchar in self.quotes: |
|---|
| 221 | self.state = nextchar |
|---|
| 222 | elif self.posix and nextchar in self.escape: |
|---|
| 223 | escapedstate = 'a' |
|---|
| 224 | self.state = nextchar |
|---|
| 225 | elif nextchar in self.wordchars or nextchar in self.quotes \ |
|---|
| 226 | or self.whitespace_split: |
|---|
| 227 | self.token = self.token + nextchar |
|---|
| 228 | else: |
|---|
| 229 | self.pushback.appendleft(nextchar) |
|---|
| 230 | if self.debug >= 2: |
|---|
| 231 | print "shlex: I see punctuation in word state" |
|---|
| 232 | self.state = ' ' |
|---|
| 233 | if self.token: |
|---|
| 234 | break # emit current token |
|---|
| 235 | else: |
|---|
| 236 | continue |
|---|
| 237 | result = self.token |
|---|
| 238 | self.token = '' |
|---|
| 239 | if self.posix and not quoted and result == '': |
|---|
| 240 | result = None |
|---|
| 241 | if self.debug > 1: |
|---|
| 242 | if result: |
|---|
| 243 | print "shlex: raw token=" + repr(result) |
|---|
| 244 | else: |
|---|
| 245 | print "shlex: raw token=EOF" |
|---|
| 246 | return result |
|---|
| 247 | |
|---|
| 248 | def sourcehook(self, newfile): |
|---|
| 249 | "Hook called on a filename to be sourced." |
|---|
| 250 | if newfile[0] == '"': |
|---|
| 251 | newfile = newfile[1:-1] |
|---|
| 252 | # This implements cpp-like semantics for relative-path inclusion. |
|---|
| 253 | if isinstance(self.infile, basestring) and not os.path.isabs(newfile): |
|---|
| 254 | newfile = os.path.join(os.path.dirname(self.infile), newfile) |
|---|
| 255 | return (newfile, open(newfile, "r")) |
|---|
| 256 | |
|---|
| 257 | def error_leader(self, infile=None, lineno=None): |
|---|
| 258 | "Emit a C-compiler-like, Emacs-friendly error-message leader." |
|---|
| 259 | if infile is None: |
|---|
| 260 | infile = self.infile |
|---|
| 261 | if lineno is None: |
|---|
| 262 | lineno = self.lineno |
|---|
| 263 | return "\"%s\", line %d: " % (infile, lineno) |
|---|
| 264 | |
|---|
| 265 | def __iter__(self): |
|---|
| 266 | return self |
|---|
| 267 | |
|---|
| 268 | def next(self): |
|---|
| 269 | token = self.get_token() |
|---|
| 270 | if token == self.eof: |
|---|
| 271 | raise StopIteration |
|---|
| 272 | return token |
|---|
| 273 | |
|---|
| 274 | def split(s, comments=False, posix=True): |
|---|
| 275 | lex = shlex(s, posix=posix) |
|---|
| 276 | lex.whitespace_split = True |
|---|
| 277 | if not comments: |
|---|
| 278 | lex.commenters = '' |
|---|
| 279 | return list(lex) |
|---|
| 280 | |
|---|
| 281 | if __name__ == '__main__': |
|---|
| 282 | if len(sys.argv) == 1: |
|---|
| 283 | lexer = shlex() |
|---|
| 284 | else: |
|---|
| 285 | file = sys.argv[1] |
|---|
| 286 | lexer = shlex(open(file), file) |
|---|
| 287 | while 1: |
|---|
| 288 | tt = lexer.get_token() |
|---|
| 289 | if tt: |
|---|
| 290 | print "Token: " + repr(tt) |
|---|
| 291 | else: |
|---|
| 292 | break |
|---|
