@@ -141,11 +141,11 @@ tokenize([$#|String], Line, Scope, Tokens) ->
141
141
142
142
tokenize ([$% ,S ,H ,H ,H |T ] = Original , Line , Scope , Tokens ) when ? is_quote (H ), ? is_upcase (S ) orelse ? is_downcase (S ) ->
143
143
case extract_heredoc_with_interpolation (Line , Scope , ? is_downcase (S ), T , H ) of
144
- { error , Reason } ->
145
- { error , Reason , Original , Tokens };
146
- { Parts , Rest } ->
144
+ { ok , NewLine , Parts , Rest } ->
147
145
{ Final , Modifiers } = collect_modifiers (Rest , []),
148
- tokenize (Final , Line , Scope , [{ sigil , Line , S , Parts , Modifiers }|Tokens ])
146
+ tokenize (Final , NewLine , Scope , [{ sigil , Line , S , Parts , Modifiers }|Tokens ]);
147
+ { error , Reason } ->
148
+ { error , Reason , Original , Tokens }
149
149
end ;
150
150
151
151
tokenize ([$% ,S ,H |T ] = Original , Line , Scope , Tokens ) when not (? is_identifier (H )), not (? is_terminator (H )), ? is_upcase (S ) orelse ? is_downcase (S ) ->
@@ -482,11 +482,6 @@ tokenize([Space, Sign, NotMarker|T], Line, Scope, [{ Identifier, _, _ } = H|Toke
482
482
483
483
tokenize ([T |Rest ], Line , Scope , Tokens ) when ? is_horizontal_space (T ) ->
484
484
tokenize (Rest , Line , Scope , Tokens );
485
-
486
- % % TODO: This token is deprecated once continuable heredocs are removed
487
- tokenize ([{line ,Line }|Rest ], _Line , Scope , Tokens ) ->
488
- tokenize (Rest , Line , Scope , Tokens );
489
-
490
485
tokenize (T , Line , _Scope , Tokens ) ->
491
486
{ error , { Line , " invalid token: " , until_eol (T ) }, T , Tokens }.
492
487
@@ -503,11 +498,11 @@ escape_char(List) ->
503
498
504
499
handle_heredocs (T , Line , H , Scope , Tokens ) ->
505
500
case extract_heredoc_with_interpolation (Line , Scope , true , T , H ) of
506
- { error , Reason } ->
507
- { error , Reason , [H , H , H ] ++ T , Tokens };
508
- { Parts , Rest } ->
501
+ { ok , NewLine , Parts , Rest } ->
509
502
Token = { string_type (H ), Line , unescape_tokens (Parts ) },
510
- tokenize (Rest , Line , Scope , [Token |Tokens ])
503
+ tokenize (Rest , NewLine , Scope , [Token |Tokens ]);
504
+ { error , Reason } ->
505
+ { error , Reason , [H , H , H ] ++ T , Tokens }
511
506
end .
512
507
513
508
handle_strings (T , Line , H , Scope , Tokens ) ->
@@ -575,34 +570,33 @@ collect_modifiers(Rest, Buffer) ->
575
570
% % Heredocs
576
571
577
572
extract_heredoc_with_interpolation (Line , Scope , Interpol , T , H ) ->
578
- case extract_heredoc (Line , T , H , Scope ) of
579
- { error , _ } = Error ->
580
- Error ;
581
- { Body , Rest } ->
573
+ case extract_heredoc (Line , T , H ) of
574
+ { ok , NewLine , Body , Rest } ->
582
575
case elixir_interpolation :extract (Line + 1 , Scope , Interpol , Body , 0 ) of
583
576
{ error , Reason } ->
584
577
{ error , interpolation_format (Reason , " (for heredoc starting at line ~B )" , [Line ]) };
585
578
{ _ , Parts , [] } ->
586
- { Parts , Rest }
587
- end
579
+ { ok , NewLine , Parts , Rest }
580
+ end ;
581
+ { error , _ } = Error ->
582
+ Error
588
583
end .
589
584
590
- extract_heredoc (Line0 , Rest0 , Marker , Scope ) ->
591
- case extract_heredoc_header (Rest0 , [], { Line0 , Scope # elixir_tokenizer . file } ) of
592
- { ok , Extra , Rest1 } ->
585
+ extract_heredoc (Line0 , Rest0 , Marker ) ->
586
+ case extract_heredoc_header (Rest0 ) of
587
+ { ok , Rest1 } ->
593
588
% % We prepend a new line so we can transparently remove
594
589
% % spaces later. This new line is removed by calling `tl`
595
590
% % in the final heredoc body three lines below.
596
591
case extract_heredoc_body (Line0 , Marker , [$\n |Rest1 ], []) of
597
- { Line1 , Body , Rest2 , Spaces } ->
598
- { tl (remove_heredoc_spaces (Body , Spaces )), merge_heredoc_extra ( Line1 , Extra , Rest2 ) };
592
+ { ok , Line1 , Body , Rest2 , Spaces } ->
593
+ { ok , Line1 , tl (remove_heredoc_spaces (Body , Spaces )), Rest2 };
599
594
{ error , ErrorLine } ->
600
595
Terminator = [Marker , Marker , Marker ],
601
596
Message = " missing terminator: ~ts (for heredoc starting at line ~B )" ,
602
597
{ error , { ErrorLine , io_lib :format (Message , [Terminator , Line0 ]), [] } }
603
598
end ;
604
599
error ->
605
- % % TODO: Test me once the deprecated continuable heredocs are removed
606
600
Terminator = [Marker , Marker , Marker ],
607
601
Message = " heredoc start ~ts must be followed by a new line" ,
608
602
{ error , { Line0 , io_lib :format (Message , [Terminator ]), [] } }
@@ -625,32 +619,25 @@ remove_heredoc_spaces([], Buffer, _Spaces, _Original) ->
625
619
626
620
% % Extract the heredoc header.
627
621
628
- extract_heredoc_header (" \r\n " ++ Rest , Buffer , _Scope ) ->
629
- { ok , Buffer , Rest };
630
- extract_heredoc_header (" \n " ++ Rest , Buffer , _Scope ) ->
631
- { ok , Buffer , Rest };
632
- extract_heredoc_header ([H |T ], Buffer , {Line ,File }) when not ? is_horizontal_space (H ) ->
633
- elixir_errors :deprecation ([{line ,Line }], File , " continuable heredocs are deprecated, parsing will no longer continue on the same line as the heredoc starts" ),
634
- extract_heredoc_header (T , [H |Buffer ], nil );
635
- extract_heredoc_header ([H |T ], Buffer , _Scope ) ->
636
- extract_heredoc_header (T , [H |Buffer ], _Scope );
637
- extract_heredoc_header (_ , _ , _Scope ) ->
622
+ extract_heredoc_header (" \r\n " ++ Rest ) ->
623
+ { ok , Rest };
624
+ extract_heredoc_header (" \n " ++ Rest ) ->
625
+ { ok , Rest };
626
+ extract_heredoc_header ([_ |T ]) ->
627
+ extract_heredoc_header (T );
628
+ extract_heredoc_header (_ ) ->
638
629
error .
639
630
640
631
% % Extract heredoc body. It returns the heredoc body (in reverse order),
641
632
% % the remaining of the document and the number of spaces the heredoc
642
633
% % is aligned.
643
634
644
- % % TODO: This token is deprecated once continuable heredocs are removed
645
- extract_heredoc_body (_Line , Marker , [{line ,NewLine }|Rest ], Buffer ) ->
646
- extract_heredoc_body (NewLine , Marker , Rest , Buffer );
647
-
648
635
extract_heredoc_body (Line , Marker , Rest , Buffer ) ->
649
636
case extract_heredoc_line (Marker , Rest , Buffer , 0 ) of
650
637
{ ok , NewBuffer , NewRest } ->
651
638
extract_heredoc_body (Line + 1 , Marker , NewRest , NewBuffer );
652
639
{ ok , NewBuffer , NewRest , Spaces } ->
653
- { Line , NewBuffer , NewRest , Spaces };
640
+ { ok , Line , NewBuffer , NewRest , Spaces };
654
641
{ error , eof } ->
655
642
{ error , Line }
656
643
end .
@@ -675,17 +662,6 @@ extract_heredoc_line(Marker, [Marker,Marker,Marker|T], Buffer, Counter) ->
675
662
extract_heredoc_line (_Marker , Rest , Buffer , _Counter ) ->
676
663
extract_heredoc_line (Rest , Buffer ).
677
664
678
- % % Merge heredoc extra by replying it on the buffer. It also adds
679
- % % a special { line, Line } token to force a line change along the way.
680
-
681
- % % TODO: This token is deprecated once continuable heredocs are removed
682
- merge_heredoc_extra (Line , Extra , Buffer ) ->
683
- merge_heredoc_extra (Extra , [{line ,Line }|Buffer ]).
684
- merge_heredoc_extra ([H |T ], Buffer ) ->
685
- merge_heredoc_extra (T , [H |Buffer ]);
686
- merge_heredoc_extra ([], Buffer ) ->
687
- Buffer .
688
-
689
665
% % Integers and floats
690
666
% % At this point, we are at least sure the first digit is a number.
691
667
0 commit comments