Author: gabor
Date: Thu Nov 24 13:20:53 2005
New Revision: 10378
Modified:
branches/line-tokenizer/parser/lexer.dylan
branches/line-tokenizer/parser/parser-exports.dylan
branches/line-tokenizer/parser/tokenize.dylan
Log:
job: 7102
move <line-tokenizer> into
module tokenize, where it belongs
Modified: branches/line-tokenizer/parser/lexer.dylan
==============================================================================
--- branches/line-tokenizer/parser/lexer.dylan (original)
+++ branches/line-tokenizer/parser/lexer.dylan Thu Nov 24 13:20:53 2005
@@ -1258,168 +1258,6 @@
-// <line-convention>
-define constant <line-convention>
- = one-of(#"unknown", #"unixy", #"mac-like", #"dosoid", #"dosoid-LF",
#"leading-CR");
-
-// <line-tokenizer> -- exported.
-//
-// An object knowing line-ending conventions.
-// assumes line and line-setter functions defined on subclasses.
-//
-define primary abstract class <line-tokenizer> (<tokenizer>)
- slot line-convention :: <line-convention>,
- init-value: #"unknown", init-keyword: convention:;
-end class <line-tokenizer>;
-
-define generic line
- (tokenizer :: <line-tokenizer>) => (line :: <integer>);
-
-define generic line-setter
- (to-line :: <integer>, tokenizer :: <line-tokenizer>) => (new-line ::
<integer>);
-
-define generic examine-ending
- (ending :: one-of('\n', '\r'),
- tokenizer :: <line-tokenizer>) => ();
-
-
-define generic examine-ending-aux
- (ending :: one-of('\n', '\r'),
- convention :: <line-convention>,
- tokenizer :: <line-tokenizer>) => ();
-
-define inline method examine-ending-aux
- (ending :: one-of('\n', '\r'),
- convention :: <line-convention>,
- tokenizer :: <line-tokenizer>) => ();
-
- tokenizer.line := tokenizer.line + 1;
-end;
-
-define inline method examine-ending-aux
- (ending == '\n',
- convention == #"dosoid-LF",
- tokenizer :: <line-tokenizer>) => ();
-
- next-method();
- tokenizer.line-convention := #"dosoid";
-end;
-
-/*
-This does not work, because lexer.posn is not correctly adjusted?
-
-// TODO: handle \rsssss\n properly by looking at last line start
-define inline method examine-ending-aux
- (ending == '\n',
- convention == #"dosoid-LF",
- lexer :: <lexer>) => ();
-
- let check :: singleton(13) = lexer.lexer-source.contents[lexer.posn - 1];
- next-method();
-end;
-*/
-
-define inline method examine-ending-aux
- (ending == '\n',
- convention == #"dosoid",
- tokenizer :: <line-tokenizer>) => ();
-
- error("dosoid file containing two line-feeds in succession?");
-end;
-
-define inline method examine-ending-aux
- (ending == '\n',
- convention == #"mac-like",
- tokenizer :: <line-tokenizer>) => ();
-
- error("mac-like file containing line-feed?");
-end;
-
-define inline method examine-ending-aux
- (ending == '\n',
- convention == #"unknown",
- tokenizer :: <line-tokenizer>) => ();
-
- tokenizer.line-convention := #"unixy";
- examine-ending(ending, tokenizer);
-end;
-
-define inline method examine-ending-aux
- (ending == '\n',
- convention == #"leading-CR",
- tokenizer :: <line-tokenizer>) => ();
-
- tokenizer.line-convention := #"dosoid"; // already incremented
-end;
-
-define method examine-ending
- (ending == '\n', tokenizer :: <line-tokenizer>)
- => ();
-
- examine-ending-aux(ending, tokenizer.line-convention, tokenizer)
-end;
-
-
-define inline method examine-ending-aux
- (ending == '\r',
- convention == #"dosoid",
- tokenizer :: <line-tokenizer>) => ();
-
- tokenizer.line-convention := #"dosoid-LF";
-end;
-
-define inline method examine-ending-aux
- (ending == '\r',
- convention == #"dosoid-LF",
- tokenizer :: <line-tokenizer>) => ();
-
- error("dosoid file with two consecutive returns?");
-end;
-
-define inline method examine-ending-aux
- (ending == '\r',
- convention == #"unixy",
- tokenizer :: <line-tokenizer>) => ();
-
- error("unixy file containing carriage-return?");
-end;
-
-define inline method examine-ending-aux
- (ending == '\r',
- convention == #"unknown",
- tokenizer :: <line-tokenizer>) => ();
-
- next-method();
- tokenizer.line-convention := #"leading-CR";
-end;
-
-define inline method examine-ending-aux
- (ending == '\r',
- convention == #"leading-CR",
- tokenizer :: <line-tokenizer>) => ();
-
- tokenizer.line-convention := #"mac-like";
- examine-ending(ending, tokenizer);
-end;
-
-
-define method examine-ending
- (ending == '\r', tokenizer :: <line-tokenizer>)
- => ();
-
- examine-ending-aux(ending, tokenizer.line-convention, tokenizer)
-end;
-
-/*
-define lout method-documentation examine-ending
- (ending == '\r', tokenizer :: <line-tokenizer>)
- => ();
- // graph for the line-ending state machine
- // TODO
-end;
-*/
-
-
// lexer
// <lexer> -- exported.
Modified: branches/line-tokenizer/parser/parser-exports.dylan
==============================================================================
--- branches/line-tokenizer/parser/parser-exports.dylan (original)
+++ branches/line-tokenizer/parser/parser-exports.dylan Thu Nov 24 13:20:53 2005
@@ -4,7 +4,7 @@
//======================================================================
//
// Copyright (c) 1995, 1996, 1997 Carnegie Mellon University
-// Copyright (c) 1998, 1999, 2000 Gwydion Dylan Maintainers
+// Copyright (c) 1998 - 2005 Gwydion Dylan Maintainers
// All rights reserved.
//
// Use and copying of this software and preparation of derivative
@@ -48,7 +48,8 @@
use source;
export
- <tokenizer>, get-token, unget-token, note-potential-end-point;
+ <tokenizer>, get-token, unget-token, note-potential-end-point,
+ <line-tokenizer>, line, line-setter, examine-ending;
end module tokenize;
define module source-utilities
Modified: branches/line-tokenizer/parser/tokenize.dylan
==============================================================================
--- branches/line-tokenizer/parser/tokenize.dylan (original)
+++ branches/line-tokenizer/parser/tokenize.dylan Thu Nov 24 13:20:53 2005
@@ -5,7 +5,7 @@
//======================================================================
//
// Copyright (c) 1995, 1996, 1997 Carnegie Mellon University
-// Copyright (c) 1998, 1999, 2000 Gwydion Dylan Maintainers
+// Copyright (c) 1998 - 2005 Gwydion Dylan Maintainers
// All rights reserved.
//
// Use and copying of this software and preparation of derivative
@@ -30,6 +30,7 @@
//======================================================================
// Tokenizer interface.
+// <tokenizer> -- exported.
define primary abstract class <tokenizer> (<object>)
end class <tokenizer>;
@@ -45,3 +46,168 @@
define method note-potential-end-point (tokenizer :: <tokenizer>) => ();
end method note-potential-end-point;
+
+
+
+// <line-convention>
+//
+define constant <line-convention>
+ = one-of(#"unknown", #"unixy", #"mac-like", #"dosoid", #"dosoid-LF",
#"leading-CR");
+
+// <line-tokenizer> -- exported.
+//
+// An object knowing line-ending conventions.
+// assumes line and line-setter functions defined on subclasses.
+//
+define primary abstract class <line-tokenizer> (<tokenizer>)
+ slot line-convention :: <line-convention>,
+ init-value: #"unknown", init-keyword: convention:;
+end class <line-tokenizer>;
+
+define generic line
+ (tokenizer :: <line-tokenizer>) => (line :: <integer>);
+
+define generic line-setter
+ (to-line :: <integer>, tokenizer :: <line-tokenizer>) => (new-line ::
<integer>);
+
+define generic examine-ending
+ (ending :: one-of('\n', '\r'),
+ tokenizer :: <line-tokenizer>) => ();
+
+
+define generic examine-ending-aux
+ (ending :: one-of('\n', '\r'),
+ convention :: <line-convention>,
+ tokenizer :: <line-tokenizer>) => ();
+
+define inline method examine-ending-aux
+ (ending :: one-of('\n', '\r'),
+ convention :: <line-convention>,
+ tokenizer :: <line-tokenizer>) => ();
+
+ tokenizer.line := tokenizer.line + 1;
+end;
+
+define inline method examine-ending-aux
+ (ending == '\n',
+ convention == #"dosoid-LF",
+ tokenizer :: <line-tokenizer>) => ();
+
+ next-method();
+ tokenizer.line-convention := #"dosoid";
+end;
+
+/*
+This does not work, because lexer.posn is not correctly adjusted?
+
+// TODO: handle \rsssss\n properly by looking at last line start
+define inline method examine-ending-aux
+ (ending == '\n',
+ convention == #"dosoid-LF",
+ lexer :: <lexer>) => ();
+
+ let check :: singleton(13) = lexer.lexer-source.contents[lexer.posn - 1];
+ next-method();
+end;
+*/
+
+define inline method examine-ending-aux
+ (ending == '\n',
+ convention == #"dosoid",
+ tokenizer :: <line-tokenizer>) => ();
+
+ error("dosoid file containing two line-feeds in succession?");
+end;
+
+define inline method examine-ending-aux
+ (ending == '\n',
+ convention == #"mac-like",
+ tokenizer :: <line-tokenizer>) => ();
+
+ error("mac-like file containing line-feed?");
+end;
+
+define inline method examine-ending-aux
+ (ending == '\n',
+ convention == #"unknown",
+ tokenizer :: <line-tokenizer>) => ();
+
+ tokenizer.line-convention := #"unixy";
+ examine-ending(ending, tokenizer);
+end;
+
+define inline method examine-ending-aux
+ (ending == '\n',
+ convention == #"leading-CR",
+ tokenizer :: <line-tokenizer>) => ();
+
+ tokenizer.line-convention := #"dosoid"; // already incremented
+end;
+
+define method examine-ending
+ (ending == '\n', tokenizer :: <line-tokenizer>)
+ => ();
+
+ examine-ending-aux(ending, tokenizer.line-convention, tokenizer)
+end;
+
+
+define inline method examine-ending-aux
+ (ending == '\r',
+ convention == #"dosoid",
+ tokenizer :: <line-tokenizer>) => ();
+
+ tokenizer.line-convention := #"dosoid-LF";
+end;
+
+define inline method examine-ending-aux
+ (ending == '\r',
+ convention == #"dosoid-LF",
+ tokenizer :: <line-tokenizer>) => ();
+
+ error("dosoid file with two consecutive returns?");
+end;
+
+define inline method examine-ending-aux
+ (ending == '\r',
+ convention == #"unixy",
+ tokenizer :: <line-tokenizer>) => ();
+
+ error("unixy file containing carriage-return?");
+end;
+
+define inline method examine-ending-aux
+ (ending == '\r',
+ convention == #"unknown",
+ tokenizer :: <line-tokenizer>) => ();
+
+ next-method();
+ tokenizer.line-convention := #"leading-CR";
+end;
+
+define inline method examine-ending-aux
+ (ending == '\r',
+ convention == #"leading-CR",
+ tokenizer :: <line-tokenizer>) => ();
+
+ tokenizer.line-convention := #"mac-like";
+ examine-ending(ending, tokenizer);
+end;
+
+
+define method examine-ending
+ (ending == '\r', tokenizer :: <line-tokenizer>)
+ => ();
+
+ examine-ending-aux(ending, tokenizer.line-convention, tokenizer)
+end;
+
+/*
+define lout method-documentation examine-ending
+ (ending == '\r', tokenizer :: <line-tokenizer>)
+ => ();
+ // graph for the line-ending state machine
+ // TODO
+end;
+*/
+
--
Gd-chatter mailing list
Gd-chatter@xxxxxxxxxxxxxxxx
https://gauss.gwydiondylan.org/mailman/listinfo/gd-chatter
|
