Add markdown support for Perl (#399)

Co-authored-by: Erik Huelsmann <[email protected]>

Author: fperrad

CHANGELOG.md

@@ -7,6 +7,9 @@ This project adheres to [Semantic Versioning](http://semver.org).
 This document is formatted according to the principles of [Keep A CHANGELOG](http://keepachangelog.com).
 
 ## [Unreleased]
+### Added
+- [Perl] Support for Gherkin in Markdown format (MDG)
+
 ### Fixed
 - [C] Prefer the longest step keyword ([#417](https://github.com/cucumber/gherkin/pull/417))
 - [cpp] Actually allow comment inside descriptions ([#414](https://github.com/cucumber/gherkin/pull/414))

perl/Makefile

@@ -8,8 +8,8 @@ SOURCE_FILES = $(shell find lib -name "*.pm" | grep -v $(GHERKIN_PARSER) | grep
 GHERKIN = bin/gherkin
 GHERKIN_GENERATE_TOKENS = bin/gherkin-generate-tokens
 
-GOOD_FEATURE_FILES = $(shell find ../testdata/good -name "*.feature")
-BAD_FEATURE_FILES  = $(shell find ../testdata/bad -name "*.feature")
+GOOD_FEATURE_FILES = $(shell find ../testdata/good -name "*.feature" -o -name "*.feature.md")
+BAD_FEATURE_FILES  = $(shell find ../testdata/bad -name "*.feature" -o -name "*.feature.md")
 
 TOKENS       = $(patsubst ../testdata/%,acceptance/testdata/%.tokens,$(GOOD_FEATURE_FILES))
 ASTS         = $(patsubst ../testdata/%,acceptance/testdata/%.ast.ndjson,$(GOOD_FEATURE_FILES))

perl/bin/gherkin-generate-tokens

@@ -6,16 +6,23 @@ use lib 'Gherkin-latest/lib';
 
 use Gherkin::Parser;
 use Gherkin::TokenFormatterBuilder;
+use Gherkin::TokenMatcher;
+use Gherkin::MarkdownTokenMatcher;
 
 package App::GherkinGenerateTokens;
 
 sub run {
     my ( $class, $fh, @file_list ) = @_;
 
-    my $parser
-        = Gherkin::Parser->new( Gherkin::TokenFormatterBuilder->new() );
-
-    print $fh join "\n", @{ $parser->parse($_) } for @file_list;
+    print $fh join "\n",
+      @{ Gherkin::Parser->new(
+            Gherkin::TokenFormatterBuilder->new(),
+            /\.md$/
+            ? Gherkin::MarkdownTokenMatcher->new()
+            : Gherkin::TokenMatcher->new()
+        )->parse($_)
+      }
+      for @file_list;
     print $fh "\n";
 
 }

perl/lib/Gherkin.pm

@@ -10,6 +10,8 @@ use Cucumber::Messages;
 use Gherkin::AstBuilder;
 use Gherkin::Parser;
 use Gherkin::Pickles::Compiler;
+use Gherkin::TokenMatcher;
+use Gherkin::MarkdownTokenMatcher;
 
 
 use Class::XSAccessor accessors =>
@@ -55,8 +57,10 @@ sub from_paths {
                 source => Cucumber::Messages::Source->new(
                     uri        => $path,
                     data       => $content,
-                    media_type => Cucumber::Messages::Source::MEDIATYPE_TEXT_X_CUCUMBER_GHERKIN_PLAIN,
-                    )
+                    media_type => $path =~ m/\.md$/
+                    ? Cucumber::Messages::Source::MEDIATYPE_TEXT_X_CUCUMBER_GHERKIN_MARKDOWN
+                    : Cucumber::Messages::Source::MEDIATYPE_TEXT_X_CUCUMBER_GHERKIN_PLAIN,
+                )
             ),
             $id_generator,
             $sink);
@@ -113,8 +117,11 @@ sub from_source {
     if ($self->include_ast or $self->include_pickles) {
         my $source = $envelope->source;
         my $parser = Gherkin::Parser->new(
-            Gherkin::AstBuilder->new($id_generator)
-            );
+            Gherkin::AstBuilder->new($id_generator),
+            $source->media_type eq Cucumber::Messages::Source::MEDIATYPE_TEXT_X_CUCUMBER_GHERKIN_MARKDOWN
+            ? Gherkin::MarkdownTokenMatcher->new()
+            : Gherkin::TokenMatcher->new()
+        );
         my $data = $source->data;
 
         local $@;

perl/lib/Gherkin/Dialect.pm

@@ -96,7 +96,8 @@ one to be used for keyword translation lookup. Out of the box, Gherkin comes
 with actual translations, such as C<Afrikaans> as well as 'slang-like'
 translations such as "Pirate English".
 
-This module is used by the L<token matcher|Gherkin::TokenMatcher> to identify
+This module is used by the L<token matcher|Gherkin::TokenMatcher> and
+the L<Markdown token matcher|Gherkin::MarkdownTokenMatcher> to identify
 the type of token (input line) passed to the scanner.
 
 =head1 METHODS