Changeset 22971 for src

Timestamp:

11/11/08 01:51:06 (2 months ago)

Author:

lwall

Message:

[Cursor] isolate symbols in a NOTBEFORE to fix mncharity++'s last; problem
[Cursor] stub in a trie-engine as down payment on a real dfa

Files:

• src/perl6/Cursor.pmc (modified) (5 diffs)

src/perl6/Cursor.pmc

@@ -13 +13 @@
 our $DEPTH = 0;
 our %LEXERS;       # per language, the cache of lexers, keyed by rule name
+
+my %HIST;
 
 sub ::init_globals {
@@ -141 +143 @@
     }
     $class->new($text)->$rule();
+#    for my $key (sort {$HIST{$a} <=> $HIST{$b}} keys(%HIST)) {
+#       warn "$HIST{$key}\t$key\n";
+#    }
 }
 
@@ -591 +596 @@
     my $retree = shift;
     # $_[0] is now ref to a $trystate;
+#    $HIST{$name}++;
 
     $self->deb("cursor_fate $pkg $name") if $DEBUG & DEBUG::cursors;
@@ -613 +619 @@
         return sub {};
     }
+
+    # A rudimentary trie walker, unused so far.  It assumes a prebuilt trie;
+    # it will need some revisions to build it on the fly.  Also, it's currently
+    # written for clarity rather than speed, and that will also change later.
+    # There is an arbitrary cutoff between characters we look up by array
+    # vs characters we look up by hash. ->{T}[0] is the array, [1] is the hash.
+    # and [2] contains actual results generator closure.  At the moment we just
+    # drop through if we don't get a result, which is always.
+
+    if (my $state = $lexer->{T}) {
+        my $p = $P;
+        my $ch = $::ORIG[$p];
+        my $next;
+        while ($next = $state->[0][$ch] // $state->[1]{chr $ch}) {
+            $ch = $::ORIG[++$p];
+            $state = $next;
+        }
+        if ($state->[2]) {
+            return $state->[2];
+        }
+    }
+
     pos($::ORIG) = $P;
     $::ORIG =~ m/\G(\[[\\<>«»]*..|[<>][<>]..|[$@%]?\w+|['"]|[ -~].|.)/smgc;
@@ -1242 +1270 @@
 
     local $CTX = $self->callm if $DEBUG & DEBUG::trace_call;
-    my @all = $block->($self);
+    my @all = $block->($self->cursor_fresh);
     return () if @all;  # XXX loses continuation
     return $self->cursor($self->{_pos})->retm();