Skip to content

Commit c713140

Browse files
glebmglebm
committed
Use Util::ascii_* in prelexer
Also removes "unicode" and replaces "nonascii". They were identical in behaviour.
1 parent 03bf13c commit c713140

File tree

4 files changed

+29
-103
lines changed

4 files changed

+29
-103
lines changed

‎src/lexer.cpp

Lines changed: 17 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <iomanip>
77
#include "lexer.hpp"
88
#include "constants.hpp"
9+
#include "util_string.hpp"
910

1011

1112
namespace Sass {
@@ -27,77 +28,14 @@ namespace Sass {
2728
const char* kwd_minus(const char* src) { return exactly<'-'>(src); };
2829
const char* kwd_slash(const char* src) { return exactly<'/'>(src); };
2930

30-
//####################################
31-
// implement some function that do exist in the standard
32-
// but those are locale aware which brought some trouble
33-
// this even seems to improve performance by quite a bit
34-
//####################################
35-
36-
bool is_alpha(const char& chr)
37-
{
38-
return unsigned(chr - 'A') <= 'Z' - 'A' ||
39-
unsigned(chr - 'a') <= 'z' - 'a';
40-
}
41-
42-
bool is_space(const char& chr)
43-
{
44-
// adapted the technique from is_alpha
45-
return chr == ' ' || unsigned(chr - '\t') <= '\r' - '\t';
46-
}
47-
48-
bool is_digit(const char& chr)
49-
{
50-
// adapted the technique from is_alpha
51-
return unsigned(chr - '0') <= '9' - '0';
52-
}
53-
54-
bool is_number(const char& chr)
55-
{
56-
// adapted the technique from is_alpha
57-
return is_digit(chr) || chr == '-' || chr == '+';
58-
}
59-
60-
bool is_xdigit(const char& chr)
61-
{
62-
// adapted the technique from is_alpha
63-
return unsigned(chr - '0') <= '9' - '0' ||
64-
unsigned(chr - 'a') <= 'f' - 'a' ||
65-
unsigned(chr - 'A') <= 'F' - 'A';
66-
}
67-
68-
bool is_punct(const char& chr)
69-
{
70-
// locale independent
71-
return chr == '.';
72-
}
73-
74-
bool is_alnum(const char& chr)
75-
{
76-
return is_alpha(chr) || is_digit(chr);
77-
}
78-
79-
// check if char is outside ascii range
80-
bool is_unicode(const char& chr)
81-
{
82-
// check for unicode range
83-
return unsigned(chr) > 127;
84-
}
85-
86-
// check if char is outside ascii range
87-
// but with specific ranges (copied from Ruby Sass)
88-
bool is_nonascii(const char& chr)
89-
{
90-
unsigned int cmp = unsigned(chr);
91-
return (
92-
(cmp >= 128 && cmp <= 15572911) ||
93-
(cmp >= 15630464 && cmp <= 15712189) ||
94-
(cmp >= 4036001920)
95-
);
31+
bool is_number(char chr) {
32+
return Util::ascii_isdigit(static_cast<unsigned char>(chr)) ||
33+
chr == '-' || chr == '+';
9634
}
9735

9836
// check if char is within a reduced ascii range
9937
// valid in a uri (copied from Ruby Sass)
100-
bool is_uri_character(const char& chr)
38+
bool is_uri_character(char chr)
10139
{
10240
unsigned int cmp = unsigned(chr);
10341
return (cmp > 41 && cmp < 127) ||
@@ -106,34 +44,33 @@ namespace Sass {
10644

10745
// check if char is within a reduced ascii range
10846
// valid for escaping (copied from Ruby Sass)
109-
bool is_escapable_character(const char& chr)
47+
bool is_escapable_character(char chr)
11048
{
11149
unsigned int cmp = unsigned(chr);
11250
return cmp > 31 && cmp < 127;
11351
}
11452

11553
// Match word character (look ahead)
116-
bool is_character(const char& chr)
54+
bool is_character(char chr)
11755
{
11856
// valid alpha, numeric or unicode char (plus hyphen)
119-
return is_alnum(chr) || is_unicode(chr) || chr == '-';
57+
return Util::ascii_isalnum(static_cast<unsigned char>(chr)) ||
58+
!Util::ascii_isascii(static_cast<unsigned char>(chr)) ||
59+
chr == '-';
12060
}
12161

12262
//####################################
12363
// BASIC CLASS MATCHERS
12464
//####################################
12565

12666
// create matchers that advance the position
127-
const char* space(const char* src) { return is_space(*src) ? src + 1 : 0; }
128-
const char* alpha(const char* src) { return is_alpha(*src) ? src + 1 : 0; }
129-
const char* unicode(const char* src) { return is_unicode(*src) ? src + 1 : 0; }
130-
const char* nonascii(const char* src) { return is_nonascii(*src) ? src + 1 : 0; }
131-
const char* digit(const char* src) { return is_digit(*src) ? src + 1 : 0; }
132-
const char* xdigit(const char* src) { return is_xdigit(*src) ? src + 1 : 0; }
133-
const char* alnum(const char* src) { return is_alnum(*src) ? src + 1 : 0; }
134-
const char* punct(const char* src) { return is_punct(*src) ? src + 1 : 0; }
135-
const char* hyphen(const char* src) { return *src && *src == '-' ? src + 1 : 0; }
136-
const char* character(const char* src) { return is_character(*src) ? src + 1 : 0; }
67+
const char* space(const char* src) { return Util::ascii_isspace(static_cast<unsigned char>(*src)) ? src + 1 : nullptr; }
68+
const char* alpha(const char* src) { return Util::ascii_isalpha(static_cast<unsigned char>(*src)) ? src + 1 : nullptr; }
69+
const char* nonascii(const char* src) { return Util::ascii_isascii(static_cast<unsigned char>(*src)) ? nullptr : src + 1; }
70+
const char* digit(const char* src) { return Util::ascii_isdigit(static_cast<unsigned char>(*src)) ? src + 1 : nullptr; }
71+
const char* xdigit(const char* src) { return Util::ascii_isxdigit(static_cast<unsigned char>(*src)) ? src + 1 : nullptr; }
72+
const char* alnum(const char* src) { return Util::ascii_isalnum(static_cast<unsigned char>(*src)) ? src + 1 : nullptr; }
73+
const char* hyphen(const char* src) { return *src == '-' ? src + 1 : 0; }
13774
const char* uri_character(const char* src) { return is_uri_character(*src) ? src + 1 : 0; }
13875
const char* escapable_character(const char* src) { return is_escapable_character(*src) ? src + 1 : 0; }
13976

‎src/lexer.hpp

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,31 +24,20 @@ namespace Sass {
2424
// BASIC CLASS MATCHERS
2525
//####################################
2626

27-
// These are locale independant
28-
bool is_space(const char& src);
29-
bool is_alpha(const char& src);
30-
bool is_punct(const char& src);
31-
bool is_digit(const char& src);
32-
bool is_number(const char& src);
33-
bool is_alnum(const char& src);
34-
bool is_xdigit(const char& src);
35-
bool is_unicode(const char& src);
36-
bool is_nonascii(const char& src);
37-
bool is_character(const char& src);
38-
bool is_uri_character(const char& src);
39-
bool escapable_character(const char& src);
27+
// Matches ASCII digits, +, and -.
28+
bool is_number(char src);
29+
30+
bool is_uri_character(char src);
31+
bool escapable_character(char src);
4032

4133
// Match a single ctype predicate.
4234
const char* space(const char* src);
4335
const char* alpha(const char* src);
4436
const char* digit(const char* src);
4537
const char* xdigit(const char* src);
4638
const char* alnum(const char* src);
47-
const char* punct(const char* src);
4839
const char* hyphen(const char* src);
49-
const char* unicode(const char* src);
5040
const char* nonascii(const char* src);
51-
const char* character(const char* src);
5241
const char* uri_character(const char* src);
5342
const char* escapable_character(const char* src);
5443

‎src/parser.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -672,7 +672,7 @@ namespace Sass {
672672
}
673673
// EO parse_include_directive
674674

675-
675+
676676
SimpleSelectorObj Parser::parse_simple_selector()
677677
{
678678
lex < css_comments >(false);
@@ -2171,7 +2171,7 @@ namespace Sass {
21712171
}
21722172

21732173
}
2174-
2174+
21752175
}
21762176

21772177
std::vector<std::string> queries;
@@ -2907,7 +2907,7 @@ namespace Sass {
29072907
}
29082908
// backup position to last significant char
29092909
while (trim && last_pos > source && last_pos < end) {
2910-
if (!Prelexer::is_space(*last_pos)) break;
2910+
if (!Util::ascii_isspace(static_cast<unsigned char>(*last_pos))) break;
29112911
utf8::prior(last_pos, source);
29122912
}
29132913

‎src/prelexer.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -336,7 +336,7 @@ namespace Sass {
336336
return alternatives<
337337
unicode_seq,
338338
alpha,
339-
unicode,
339+
nonascii,
340340
exactly<'-'>,
341341
exactly<'_'>,
342342
NONASCII,
@@ -351,7 +351,7 @@ namespace Sass {
351351
return alternatives<
352352
unicode_seq,
353353
alnum,
354-
unicode,
354+
nonascii,
355355
exactly<'-'>,
356356
exactly<'_'>,
357357
NONASCII,
@@ -385,7 +385,7 @@ namespace Sass {
385385
{
386386
return alternatives <
387387
alpha,
388-
unicode,
388+
nonascii,
389389
escape_seq,
390390
exactly<'_'>
391391
>(src);
@@ -395,7 +395,7 @@ namespace Sass {
395395
{
396396
return alternatives <
397397
alnum,
398-
unicode,
398+
nonascii,
399399
escape_seq,
400400
exactly<'_'>
401401
>(src);

0 commit comments

Comments
 (0)