;+ ; ;Procedure: lex ; ;Purpose: Turns a string into a list of token structs with some basic ; processing performed, for the mini language. The tokens are ; generated by get_token. The processing performed is to remove blanks ; and check for and illegal token types(token types supported by IDL but ; not the mini language, like @statements) ; ; Inputs: s: the string to be tokenized ; ; Keywords: token_list: The token_list is returned through this variable ; error: If an error occurs, a struct describing it will be returned through this variable ; ; $LastChangedBy: pcruce $ ; $LastChangedDate: 2013-01-09 12:57:39 -0800 (Wed, 09 Jan 2013) $ ; $LastChangedRevision: 11408 $ ; $URL: svn+ssh://thmsvn@ambrosia.ssl.berkeley.edu/repos/spdsoft/trunk/general/mini/lex.pro $ ; pro lex,s,token_list=token_list,error=error compile_opt idl2 mini_routines ;compiles procedures associated with library routines for the mini language mini_predicates ;compiles predicates for mini language n = 0 len = strlen(s) error = 0 if keyword_set(token_list) then begin t = temporary(token_list) endif if keyword_set(error) then begin t = temporary(error) endif repeat begin if keyword_set(token) && ~is_blank_type(token) then begin previous = token endif token = get_token(strmid(s,n,len-n)) n += strlen(token.value) ;this set of ifs transforms the tokens into types for use with the parser if is_invalid_type(token) then begin error = {type:token.type,name:token.name,value:token.value,position:n} return endif if ~is_blank_type(token) then begin if ~is_struct(token_list) then begin token_list = [token] endif else begin token_list = [token_list,token] endelse endif endrep until is_endline_type(token) end