(***********************************************************************) (* *) (* Copyright 2006 Philippe Wang - mail@philippewang.info *) (* *) (* Yet Another Caml Lexer *) (* *) (* This program is free software; you can redistribute it and/or *) (* modify it under the terms of the GNU General Public *) (* License as published by the Free Software Foundation; either *) (* version 2 of the License, or (at your option) any later version. *) (* *) (* This program is distributed in the hope that it will be useful, *) (* but WITHOUT ANY WARRANTY; without even the implied warranty of *) (* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU *) (* General Public License for more details. *) (* *) (* You should have received a copy of the GNU General Public *) (* License along with this program; if not, write to the *) (* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, *) (* Boston, MA 02110-1301, USA. *) (***********************************************************************) open Lexing2 let error loc m = Printf.printf "Characters %d:%d - %d:%d :\nError: %s\n" loc.sl loc.sc loc.el loc.ec m; exit 42 let print t = prerr_string ("(" ^ string_of_int t.loc.sl ^ "," ^ string_of_int t.loc.sc ^ ") -> (" ^ string_of_int t.loc.el ^ "," ^ string_of_int t.loc.ec ^ ") "); begin match t.token with Int i -> prerr_string ("Int " ^ string_of_int i) | Float f -> prerr_string ("Float " ^ string_of_float f) | Upper s -> prerr_string ("Upper " ^ s) | Lower s -> prerr_string ("Lower " ^ s) | String s -> prerr_string ("String \"" ^ s ^ "\"") | Char c -> prerr_string "Char \'"; prerr_char c; prerr_string "\'" | Infix s -> prerr_string ("Infix (" ^ s ^ ")") | Prefix s -> prerr_string ("Prefix (" ^ s ^ ")") | Operator s -> prerr_string ("Operator (" ^ s ^ ")") | Comment s -> prerr_string ("Comment " ^ s) | And -> prerr_string "And" | As -> prerr_string "As" | Begin -> prerr_string "Begin" | Class -> prerr_string "Class" | Constraint -> prerr_string "Constraint" | Do -> prerr_string "Do" | Done -> prerr_string "Done" | Downto -> prerr_string "Downto" | Else -> prerr_string "Else" | End -> prerr_string "End" | Exception -> prerr_string "Exception" | External -> prerr_string "External" | False -> prerr_string "False" | For -> prerr_string "For" | Fun -> prerr_string "Fun" | Function -> prerr_string "Function" | Functor -> prerr_string "Functor" | If -> prerr_string "If" | In -> prerr_string "In" | Include -> prerr_string "Include" | Inherit -> prerr_string "Inherit" | Initializer -> prerr_string "Initializer" | Lazy -> prerr_string "Lazy" | Let -> prerr_string "Let" | Match -> prerr_string "Match" | Method -> prerr_string "Method" | Mod -> prerr_string "Mod" | Module -> prerr_string "Module" | Mutable -> prerr_string "Mutable" | New -> prerr_string "New" | Object -> prerr_string "Object" | Of -> prerr_string "Of" | Open -> prerr_string "Open" | Private -> prerr_string "Private" | Rec -> prerr_string "Rec" | Sig -> prerr_string "Sig" | Struct -> prerr_string "Struct" | Then -> prerr_string "Then" | To -> prerr_string "To" | True -> prerr_string "True" | Try -> prerr_string "Try" | Type -> prerr_string "Type" | Val -> prerr_string "Val" | Virtual -> prerr_string "Virtual" | When -> prerr_string "When" | While -> prerr_string "While" | With -> prerr_string "With" | Abstract -> prerr_string "Abstract" | EpEq -> prerr_string "EpEq" | Sharp -> prerr_string "Sharp" | Amp -> prerr_string "Amp" | AmpAmp -> prerr_string "AmpAmp" | Quote -> prerr_string "Quote" | OpenP -> prerr_string "OpenP" | CloseP -> prerr_string "CloseP" | Star -> prerr_string "Star" | Plus -> prerr_string "Plus" | Comma -> prerr_string "Comma" | Hyphen -> prerr_string "Hyphen" | HyphenDot -> prerr_string "HyphenDot" | HyphenGt -> prerr_string "HyphenGt" | Dot -> prerr_string "Dot" | DotDot -> prerr_string "DotDot" | Colon -> prerr_string "Colon" | ColonColon -> prerr_string "ColonColon" | ColonEq -> prerr_string "ColonEq" | ColonGt -> prerr_string "ColonGt" | SemiColon -> prerr_string "SemiColon" | SemiColonSemiColon -> prerr_string "SemiColonSemiColon" | Lt -> prerr_string "Lt" | LtHyphen -> prerr_string "LtHyphen" | Eq -> prerr_string "Eq" | Gt -> prerr_string "Gt" | GtCloseSB -> prerr_string "GtCloseSB" | GtCloseCB -> prerr_string "GtCloseCB" | Im -> prerr_string "Im" | ImIm -> prerr_string "ImIm" | OpenSB -> prerr_string "OpenSB" | OpenSBLt -> prerr_string "OpenSBLt" | OpenSBGt -> prerr_string "OpenSBGt" | OpenSBVertLine -> prerr_string "OpenSBVertLine" | CloseSB -> prerr_string "CloseSB" | Underscore -> prerr_string "Underscore" | BackQuote -> prerr_string "BackQuote" | OpenCB -> prerr_string "OpenCB" | OpenCBLt -> prerr_string "OpenCBLt" | VertLine -> prerr_string "VertLine" | VertLineVertLine -> prerr_string "VertLineVertLine" | VertLineCloseSB -> prerr_string "VertLineCloseSB" | CloseCB -> prerr_string "CloseCB" | Tild -> prerr_string "Tild" | Lor -> prerr_string "Lor" | Lsl -> prerr_string "Lsl" | Lsr -> prerr_string "Lsr" | Asr -> prerr_string "Asr" | Or -> prerr_string "Or" | Lxor -> prerr_string "Lxor" | Xor -> prerr_string "xor" | Eof -> prerr_string "EOF" | Land -> prerr_string "Land" end; prerr_newline () let create_buffer_from_channel channel = let rec x = {buffer = []; stream = channel; widths = []; line = 1; column = 0; pop = (fun () -> match x.buffer with [] -> begin match input_char x.stream with '\n' as c -> x.widths <- x.column :: x.widths; x.line <- x.line + 1; x.column <- 0; c | c -> x.column <- x.column + 1; c end | c :: tl -> match c with '\n' -> x.widths <- x.column :: x.widths; x.line <- x.line + 1; x.column <- 0; x.buffer <- tl; c | _ -> x.column <- x.column + 1; x.buffer <- tl; c); push = fun c -> begin match c with '\n' -> x.column <- List.hd x.widths; x.widths <- List.tl x.widths; x.line <- x.line - 1 | _ -> x.column <- x.column - 1 end; match x.buffer with [] -> x.buffer <- [c] | l -> x.buffer <- c :: l} in x let read_number stream = let loc = {sl = stream.line; sc = stream.column; el = stream.line; ec = stream.column} in let string_of_char c = let s = " " in s.[0] <- c; s in let float = ref false in let int = ref false in let dot = ref false in let rec f res = try match stream.pop () with '0' -> begin match stream.pop () with 'O' | 'o' | 'x' | 'X' | 'b' | 'B' as c -> if res <> "" or !float or !int then error {loc with el = stream.line; ec = stream.column} "Lexical error: you gave a weird number" else int := true; f ("0" ^ string_of_char c) | '0'..'9' as c -> f (res ^ "0" ^ string_of_char c) | c -> stream.push c; f (res ^ "0") end | '1'..'9' as c -> f (res ^ string_of_char c) | 'E' | 'e' as e -> if res = "" or !float or !int then begin stream.push e; res end else begin float := true; match stream.pop () with '0'..'9' as c -> f (res ^ "e" ^ string_of_char c) | c -> stream.push c; stream.push e; res end | '.' as c -> if res = "" or !float or !int then error {loc with el = stream.line; ec = stream.column} "Lexical error: you gave a weird number" else dot := true; f (res ^ string_of_char c) | '_' -> f res | c -> stream.push c; res with End_of_file -> res in let r = f "" in let n = if !float or !dot then Float (try float_of_string r with _ -> error loc "Lexical error while trying to read a float") else Int (try int_of_string r with _ -> Printf.printf "2(%d:%d) <%s>\n" stream.line stream.column r; error loc "Lexical error while trying to read an integer") in {token = n; loc = {loc with el = stream.line; ec = stream.column}} let read_upper stream = let loc = {sl = stream.line; sc = stream.column; el = stream.line; ec = stream.column} in let string_of_char c = let s = " " in s.[0] <- c; s in let rec f res = try match stream.pop () with 'A'..'Z' as c -> f (res ^ string_of_char c) | 'a'..'z' | '0'..'9' | '_' as c -> if res = "" then failwith "bad use of lexer (read_upper)"; f (res ^ string_of_char c) | c -> stream.push c; res with End_of_file -> res in let res = f "" in {token = Upper res; loc = {loc with el = stream.line; ec = stream.column}} let read_lower stream = let loc = {sl = stream.line; sc = stream.column; el = stream.line; ec = stream.column} in let string_of_char c = let s = " " in s.[0] <- c; s in let rec f res = try match stream.pop () with 'a'..'z' | '_' as c -> f (res ^ string_of_char c) | 'A'..'Z' | '0'..'9' as c -> if res = "" then failwith "bad use of lexer (read_lower)"; f (res ^ string_of_char c) | c -> stream.push c; res with End_of_file -> res in let res = f "" in try {token = List.assoc res keywords; loc = {loc with el = stream.line; ec = stream.column}} with Not_found -> try {token = List.assoc res keyops; loc = {loc with el = stream.line; ec = stream.column}} with Not_found -> {token = Lower res; loc = {loc with el = stream.line; ec = stream.column}} let read_infix stream = let loc = {sl = stream.line; sc = stream.column; el = stream.line; ec = stream.column} in let string_of_char c = let s = " " in s.[0] <- c; s in let rec f res = try match stream.pop () with '!' | '?' | '~' as c -> if res = "" then failwith "bad use of lexer (read_infix)"; f (res ^ string_of_char c) | '$' | '%' | '&' | '*' | '+' | '-' | '.' | '/' | ':' | '<' | '=' | '>' | '@' | '^' | '|' as c -> f (res ^ string_of_char c) | ']' | '}' as c -> if List.mem_assoc (res ^ string_of_char c) keyops then f_spe (res ^ string_of_char c) else begin stream.push c; res end | c -> stream.push c; res with End_of_file -> res and f_spe res = try match stream.pop () with '!' | '&' | '*' | '+' | '-' | '.' | ':' | '<' | '=' | '>' | '?' | '~' | '`' | '(' | ')' | '#' | '[' | ']' | '{' | '|' | '}' as c -> if res = "" || List.mem_assoc (res ^ string_of_char c) keyops then f (res ^ string_of_char c) else begin stream.push c; res end | c -> stream.push c; res with End_of_file -> res in let res = f "" in try {token = List.assoc res keyops; loc = {loc with el = stream.line; ec = stream.column}} with Not_found -> {token = Infix res; loc = {loc with el = stream.line; ec = stream.column}} let read_prefix stream = let loc = {sl = stream.line; sc = stream.column; el = stream.line; ec = stream.column} in let string_of_char c = let s = " " in s.[0] <- c; s in let rec f res = try match stream.pop () with '!' | '?' | '~' as c -> if res <> "" then failwith "bad use of lexer (read_prefix)"; f (res ^ string_of_char c) | '$' | '%' | '&' | '*' | '+' | '-' | '.' | '/' | ':' | '<' | '=' | '>' | '@' | '^' | '|' as c -> f (res ^ string_of_char c) | c -> stream.push c; res with End_of_file -> res in let res = f "" in try {token = List.assoc res keyops; loc = {loc with el = stream.line; ec = stream.column}} with Not_found -> {token = Prefix res; loc = {loc with el = stream.line; ec = stream.column}} let read_char stream = let loc = {sl = stream.line; sc = stream.column; el = stream.line; ec = stream.column} in let rec int i lvl = match stream.pop () with '0'..'9' as c -> if lvl > 2 then error {loc with el = stream.line; ec = stream.column} "Character decimal notation too long" else int (i * 10 + Char.code c - Char.code '0') (lvl + 1) | '\'' -> {token = Char (char_of_int i); loc = {loc with el = stream.line; ec = stream.column}} | c -> error {loc with el = stream.line; ec = stream.column} "Lexical error while trying to read an escaped ascii char" in let rec hexa i = match stream.pop () with '0'..'9' as c -> hexa (i * 16 + Char.code c - Char.code '0') | 'a'..'f' as c -> hexa (i * 16 + Char.code c - Char.code 'a' + 10) | 'A'..'F' as c -> hexa (i * 16 + Char.code c - Char.code 'A' + 10) | ';' -> if stream.pop () <> '\'' then error {loc with el = stream.line; ec = stream.column} "Lexical error while trying to read an escaped unicode char" else {token = Char (char_of_int i); loc = {loc with el = stream.line; ec = stream.column}} | c -> error {loc with el = stream.line; ec = stream.column} "Lexical error while trying to read an escaped unicode char" in match stream.pop () with '\'' -> begin match stream.pop () with '\\' -> begin match stream.pop () with '\'' -> if stream.pop () = '\'' then {token = Char '\''; loc = {loc with el = stream.line; ec = stream.column}} else error {loc with el = stream.line; ec = stream.column} "Lexical error: Illegal backslash escape in character" | 'n' -> if stream.pop () = '\'' then {token = Char '\n'; loc = {loc with el = stream.line; ec = stream.column}} else error {loc with el = stream.line; ec = stream.column} "Lexical error: Illegal backslash escape in character" | 'b' -> if stream.pop () = '\'' then {token = Char '\b'; loc = {loc with el = stream.line; ec = stream.column}} else error {loc with el = stream.line; ec = stream.column} "Lexical error: Illegal backslash escape in character" | 'r' -> if stream.pop () = '\'' then {token = Char '\r'; loc = {loc with el = stream.line; ec = stream.column}} else error {loc with el = stream.line; ec = stream.column} "Lexical error: Illegal backslash escape in character" | 't' -> if stream.pop () = '\'' then {token = Char '\t'; loc = {loc with el = stream.line; ec = stream.column}} else error {loc with el = stream.line; ec = stream.column} "Lexical error: Illegal backslash escape in character" | '0'..'9' as c -> stream.push c; int 0 0 | '#' -> hexa 0 | '\\' -> if stream.pop () = '\'' then {token = Char '\\'; loc = {loc with el = stream.line; ec = stream.column}} else error {loc with el = stream.line; ec = stream.column} "Lexical error: Illegal backslash escape in character" | c -> error {loc with el = stream.line; ec = stream.column} "Lexical error: Illegal backslash escape in character" end | c -> let c2 = stream.pop () in if c2 = '\'' then {token = Char c; loc = {loc with el = stream.line; ec = stream.column}} else begin stream.push c2; stream.push c; {token = Quote; loc = {loc with el = stream.line; ec = stream.column}} end end | _ -> assert false let read_string stream = let res = let loc = {sl = stream.line; sc = stream.column; el = stream.line; ec = stream.column} in let string_of_char c = let s = " " in s.[0] <- c; s in let rec int i lvl = match stream.pop () with '0'..'9' as c -> if lvl = 2 then string_of_char (Char.chr (i * 10 + Char.code c - Char.code '0')) else int (i * 10 + Char.code c - Char.code '0') (lvl + 1) | '\"' | '\\' as c -> stream.push c; string_of_char (Char.chr i) | c -> error {loc with el = stream.line; ec = stream.column} "Lexical error while trying to read an escaped ascii char" in let rec hexa i = match stream.pop () with '0'..'9' as c -> hexa (i * 16 + Char.code c - Char.code '0') | 'a'..'f' as c -> hexa (i * 16 + Char.code c - Char.code 'a' + 10) | 'A'..'F' as c -> hexa (i * 16 + Char.code c - Char.code 'A' + 10) | ';' -> string_of_char (Char.chr i) | c -> error {loc with el = stream.line; ec = stream.column} "Lexical error while trying to read an escaped unicode char" in let rec f res = match stream.pop () with '\"' -> res | '\\' -> begin match stream.pop () with '\"' -> f (res ^ "\"") | 'n' -> f (res ^ "\n") | 'b' -> f (res ^ "\b") | 'r' -> f (res ^ "\r") | 't' -> f (res ^ "\t") | '\\' -> f (res ^ "\\") | '0'..'9' as c -> stream.push c; f (res ^ int 0 0) | '#' -> f (res ^ hexa 0) | c -> error {loc with el = stream.line; ec = stream.column} "Lexical error: Illegal backslash escape in string" end | c -> f (res ^ string_of_char c) in match stream.pop () with '\"' -> {token = String (f ""); loc = {loc with el = stream.line; ec = stream.column}} | _ -> assert false in res let read_comment stream = let loc = {sl = stream.line; sc = stream.column; el = stream.line; ec = stream.column} in let string_of_char c = let s = " " in s.[0] <- c; s in let rec f res level = match stream.pop () with '(' -> begin match stream.pop () with '*' -> f (res ^ "(*") (level + 1) | c -> f (res ^ "(" ^ string_of_char c) level end | '*' -> begin match stream.pop () with ')' -> if level = 1 then res ^ "*)" else f (res ^ "*)") (level - 1) | '*' -> stream.push '*'; f (res ^ "*") level | c -> f (res ^ "*" ^ string_of_char c) level end | c -> f (res ^ string_of_char c) level in {token = Comment (f "" 0); loc = {loc with el = stream.line; ec = stream.column}} let read_op stream = let loc = {sl = stream.line; sc = stream.column; el = stream.line; ec = stream.column} in let string_of_char c = let s = " " in s.[0] <- c; s in let rec f res = try match stream.pop () with '!' | '&' | '*' | '+' | '-' | '.' | ':' | '<' | '=' | '>' | '?' | '~' as c -> f (res ^ string_of_char c) | '[' | ']' | '{' | '|' | '}' as c -> if res = "" || List.mem_assoc (res ^ string_of_char c) keyops then f_spe (res ^ string_of_char c) else begin stream.push c; res end | '`' | '(' | ')' | '#' as c -> if res = "" then string_of_char c else begin stream.push c; res end | ',' -> if res = "" then "," else begin stream.push ','; res end | ';' -> if res = "" then match stream.pop () with ';' -> ";;" | c -> stream.push c; ";" else begin stream.push ';'; res end | c -> stream.push c; res with End_of_file -> res and f_spe res = try match stream.pop () with '!' | '&' | '*' | '+' | '-' | '.' | ':' | '<' | '=' | '>' | '?' | '~' | '`' | '(' | ')' | '#' | '[' | ']' | '{' | '|' | '}' as c -> if res = "" || List.mem_assoc (res ^ string_of_char c) keyops then f (res ^ string_of_char c) else begin stream.push c; res end | c -> stream.push c; res with End_of_file -> res in let res = f "" in try {token = List.assoc res keyops; loc = {loc with el = stream.line; ec = stream.column}} with Not_found -> {token = Operator res; loc = {loc with el = stream.line; ec = stream.column}} let read stream = let rec f res = try match stream.pop () with '0'..'9' as c -> stream.push c; f (read_number stream :: res) | 'A'..'Z' as c -> stream.push c; f (read_upper stream :: res) | 'a'..'z' | '_' as c -> stream.push c; f (read_lower stream :: res) | ' ' | '\t' | '\n' | '\r' -> f res | '!' | '?' | '~' as c -> stream.push c; f (read_prefix stream :: res) | '=' | '<' | '>' | '@' | '^' | '|' | '&' | '+' | '-' | '*' | '/' | '$' | '%' as c -> stream.push c; f (read_infix stream :: res) | '(' -> begin match stream.pop () with '*' -> stream.push '*'; stream.push '('; f (read_comment stream :: res) | c -> stream.push c; stream.push '('; f (read_op stream :: res) end | '\'' as c -> stream.push c; f (read_char stream :: res) | '\"' as c -> stream.push c; f (read_string stream :: res) | '\\' -> error {sl = stream.line; sc = stream.column - 1; el = stream.line; ec = stream.column} "Lexical error: unexpected presence of a backslash" | c -> stream.push c; f (read_op stream :: res) with End_of_file -> res in List.rev (f []) let read_one stream = let rec f res = match res with [e] -> e | _ :: _ -> failwith "error : file lexer.ml, function read_one" | [] -> try match stream.pop () with '0'..'9' as c -> stream.push c; f (read_number stream :: res) | 'A'..'Z' as c -> stream.push c; f (read_upper stream :: res) | 'a'..'z' | '_' as c -> stream.push c; f (read_lower stream :: res) | ' ' | '\t' | '\n' | '\r' -> f res | '!' | '?' | '~' as c -> stream.push c; f (read_prefix stream :: res) | '=' | '<' | '>' | '@' | '^' | '|' | '&' | '+' | '-' | '*' | '/' | '$' | '%' as c -> stream.push c; f (read_infix stream :: res) | '(' -> begin match stream.pop () with '*' -> stream.push '*'; stream.push '('; f (read_comment stream :: res) | c -> stream.push c; stream.push '('; f (read_op stream :: res) end | '\'' as c -> stream.push c; f (read_char stream :: res) | '\"' as c -> stream.push c; f (read_string stream :: res) | '\\' -> error {sl = stream.line; sc = stream.column - 1; el = stream.line; ec = stream.column} "Lexical error: unexpected presence of a backslash" | c -> stream.push c; f (read_op stream :: res) with End_of_file -> {token = Eof; loc = {el = stream.line; ec = stream.column; sl = stream.line; sc = stream.column}} in f [] (* (* example *) let _ = let flow = read (create_buffer_from_channel (open_in Sys.argv.(1))) in List.iter print flow *) let from_channel : string -> in_channel -> lexbuf = fun name ic -> {lex_start_p = {name = name; lin = 0; col = 0}; lex_curr_p = {name = name; lin = 0; col = 0}; lex_buf = create_buffer_from_channel ic} let parser_token_of_lexing_token = function Int int -> Parser.INT int | Float float -> Parser.FLOAT float | Upper string -> Parser.UIDENT string | Lower string -> Parser.LIDENT string | String string -> Parser.STRING string | Char char -> Parser.CHAR char | Infix string -> Parser.INFIXOP4 string | Prefix string -> Parser.PREFIXOP string | Operator string -> Parser.INFIXOP4 string | Comment string -> failwith ("Comment: " ^ string) | And -> Parser.AND | As -> Parser.AS | Begin -> Parser.BEGIN | Class -> assert false | Constraint -> assert false | Do -> Parser.DO | Done -> Parser.DONE | Downto -> Parser.DOWNTO | Else -> Parser.ELSE | End -> Parser.END | Exception -> Parser.EXCEPTION | External -> Parser.EXTERNAL | False -> Parser.FALSE | For -> Parser.FOR | Fun -> Parser.FUN | Function -> Parser.FUNCTION | Functor -> assert false | If -> Parser.IF | In -> Parser.IN | Include -> assert false | Inherit -> assert false | Initializer -> assert false | Lazy -> assert false | Let -> Parser.LET | Match -> Parser.MATCH | Method -> assert false | Mod -> Parser.MOD | Module -> assert false | Mutable -> Parser.MUTABLE | New -> assert false | Object -> assert false | Of -> Parser.OF | Open -> Parser.OPEN | Private -> Parser.PRIVATE | Rec -> Parser.REC | Sig -> assert false | Struct -> assert false | Then -> Parser.THEN | To -> Parser.TO | True -> Parser.TRUE | Try -> Parser.TRY | Type -> Parser.TYPE | Val -> Parser.VAL | Virtual -> assert false | When -> Parser.WHEN | While -> Parser.WHILE | With -> Parser.WITH | Abstract -> Parser.ABSTRACT | EpEq -> Parser.INFIXOP4 "!=" | Sharp -> assert false | Amp -> Parser.AMPERSAND | AmpAmp -> Parser.AMPERAMPER | Quote -> Parser.QUOTE | OpenP -> Parser.LPAREN | CloseP -> Parser.RPAREN | Star -> Parser.STAR | Plus -> Parser.PLUS | Comma -> Parser.COMMA | Hyphen -> Parser.MINUS | HyphenDot -> Parser.MINUSDOT | HyphenGt -> Parser.MINUSGREATER | Dot -> Parser.DOT | DotDot -> Parser.DOTDOT | Colon -> Parser.COLON | ColonColon -> Parser.COLONCOLON | ColonEq -> Parser.COLONEQUAL | ColonGt -> assert false | SemiColon -> Parser.SEMI | SemiColonSemiColon -> Parser.SEMISEMI | Lt -> Parser.LESS | LtHyphen -> Parser.LESSMINUS | Eq -> Parser.EQUAL | Gt -> Parser.GREATER | GtCloseSB -> assert false | GtCloseCB -> assert false | Im -> assert false | ImIm -> assert false | OpenSB -> Parser.LBRACKET | OpenSBLt -> assert false | OpenSBGt -> assert false | OpenSBVertLine -> Parser.LBRACKETBAR | CloseSB -> Parser.RBRACKET | Underscore -> Parser.UNDERSCORE | BackQuote -> assert false | OpenCB -> Parser.LBRACE | OpenCBLt -> assert false | VertLine -> Parser.BAR | VertLineVertLine -> Parser.BARBAR | VertLineCloseSB -> Parser.BARRBRACKET | CloseCB -> Parser.RBRACE | Tild -> assert false | Lor -> Parser.INFIXOP4 "lor" | Lsl -> Parser.INFIXOP4 "lsl" | Lsr -> Parser.INFIXOP4 "lsr" | Asr -> Parser.INFIXOP4 "asr" | Or -> Parser.INFIXOP4 "or" | Lxor -> Parser.INFIXOP4 "lxor" | Land -> Parser.INFIXOP4 "land" | Xor -> Parser.INFIXOP4 "xor" | Eof -> Parser.EOF let rec get_token_from_lexbuf lexbuf = let token = read_one lexbuf.lex_buf in lexbuf.lex_start_p <- {(lexbuf.lex_start_p) with lin = token.loc.sl; col = token.loc.sc}; lexbuf.lex_curr_p <- {(lexbuf.lex_curr_p) with lin = token.loc.el; col = token.loc.ec}; match token.token with Comment _ -> get_token_from_lexbuf lexbuf | _ -> parser_token_of_lexing_token token.token