[Perl] add MarkdownTokenMatcher skeleton

Author: fperrad

perl/bin/gherkin-generate-tokens

@@ -6,16 +6,23 @@ use lib 'Gherkin-latest/lib';
 
 use Gherkin::Parser;
 use Gherkin::TokenFormatterBuilder;
+use Gherkin::TokenMatcher;
+use Gherkin::MarkdownTokenMatcher;
 
 package App::GherkinGenerateTokens;
 
 sub run {
     my ( $class, $fh, @file_list ) = @_;
 
-    my $parser
-        = Gherkin::Parser->new( Gherkin::TokenFormatterBuilder->new() );
-
-    print $fh join "\n", @{ $parser->parse($_) } for @file_list;
+    print $fh join "\n",
+      @{ Gherkin::Parser->new(
+            Gherkin::TokenFormatterBuilder->new(),
+            /\.md$/
+            ? Gherkin::MarkdownTokenMatcher->new()
+            : Gherkin::TokenMatcher->new()
+        )->parse($_)
+      }
+      for @file_list;
     print $fh "\n";
 
     return;

perl/lib/Gherkin.pm

@@ -10,6 +10,8 @@ use Cucumber::Messages;
 use Gherkin::AstBuilder;
 use Gherkin::Parser;
 use Gherkin::Pickles::Compiler;
+use Gherkin::TokenMatcher;
+use Gherkin::MarkdownTokenMatcher;
 
 
 use Class::XSAccessor accessors =>
@@ -55,8 +57,10 @@ sub from_paths {
                 source => Cucumber::Messages::Source->new(
                     uri        => $path,
                     data       => $content,
-                    media_type => Cucumber::Messages::Source::MEDIATYPE_TEXT_X_CUCUMBER_GHERKIN_PLAIN,
-                    )
+                    media_type => $path =~ m/\.md$/
+                    ? Cucumber::Messages::Source::MEDIATYPE_TEXT_X_CUCUMBER_GHERKIN_MARKDOWN
+                    : Cucumber::Messages::Source::MEDIATYPE_TEXT_X_CUCUMBER_GHERKIN_PLAIN,
+                )
             ),
             $id_generator,
             $sink);
@@ -115,8 +119,11 @@ sub from_source {
     if ($self->include_ast or $self->include_pickles) {
         my $source = $envelope->source;
         my $parser = Gherkin::Parser->new(
-            Gherkin::AstBuilder->new($id_generator)
-            );
+            Gherkin::AstBuilder->new($id_generator),
+            $source->media_type eq Cucumber::Messages::Source::MEDIATYPE_TEXT_X_CUCUMBER_GHERKIN_MARKDOWN
+            ? Gherkin::MarkdownTokenMatcher->new()
+            : Gherkin::TokenMatcher->new()
+        );
         my $data = $source->data;
 
         local $@;

perl/lib/Gherkin/MarkdownTokenMatcher.pm

@@ -0,0 +1,8 @@
+package Gherkin::MarkdownTokenMatcher;
+
+use strict;
+use warnings;
+
+use base 'Gherkin::TokenMatcher';
+
+1;