Merge pull request #9 from daohoangson/fix_chars_need_escaping

Update urlchar to handle character escaping.

Author: kisielk

scanner/scanner.go

@@ -114,10 +114,18 @@ var macros = map[string]string{
 	"num":        `[0-9]*\.[0-9]+|[0-9]+`,
 	"string":     `"(?:{stringchar}|')*"|'(?:{stringchar}|")*'`,
 	"stringchar": `{urlchar}|[ ]|\\{nl}`,
-	"urlchar":    "[\u0009\u0021\u0023-\u0026\u0027-\u007E]|{nonascii}|{escape}",
 	"nl":         `[\n\r\f]|\r\n`,
 	"w":          `{wc}*`,
 	"wc":         `[\t\n\f\r ]`,
+
+	// urlchar should accept [(ascii characters minus those that need escaping)|{nonascii}|{escape}]
+	// ASCII characters range = `[\u0020-\u007e]`
+	// Skip space \u0020 = `[\u0021-\u007e]`
+	// Skip quotation mark \0022 = `[\u0021\u0023-\u007e]`
+	// Skip apostrophe \u0027 = `[\u0021\u0023-\u0026\u0028-\u007e]`
+	// Skip reverse solidus \u005c = `[\u0021\u0023-\u0026\u0028-\u005b\u005d\u007e]`
+	// Finally, the left square bracket (\u005b) and right (\u005d) needs escaping themselves
+	"urlchar": "[\u0021\u0023-\u0026\u0028-\\\u005b\\\u005d-\u007E]|{nonascii}|{escape}",
 }
 
 // productions maps the list of tokens to patterns to be expanded.
@@ -254,10 +262,10 @@ func (s *Scanner) Next() *Token {
 		match := matchers[TokenString].FindString(input)
 		if match != "" {
 			return s.emitToken(TokenString, match)
-		} else {
-			s.err = &Token{TokenError, "unclosed quotation mark", s.row, s.col}
-			return s.err
 		}
+
+		s.err = &Token{TokenError, "unclosed quotation mark", s.row, s.col}
+		return s.err
 	case '/':
 		// Comment, error or Char.
 		if len(input) > 1 && input[1] == '*' {

scanner/scanner_test.go

@@ -31,7 +31,13 @@ func TestMatchers(t *testing.T) {
 
 	checkMatch("abcd", TokenIdent, "abcd")
 	checkMatch(`"abcd"`, TokenString, `"abcd"`)
+	checkMatch(`"ab'cd"`, TokenString, `"ab'cd"`)
+	checkMatch(`"ab\"cd"`, TokenString, `"ab\"cd"`)
+	checkMatch(`"ab\\cd"`, TokenString, `"ab\\cd"`)
 	checkMatch("'abcd'", TokenString, "'abcd'")
+	checkMatch(`'ab"cd'`, TokenString, `'ab"cd'`)
+	checkMatch(`'ab\'cd'`, TokenString, `'ab\'cd'`)
+	checkMatch(`'ab\\cd'`, TokenString, `'ab\\cd'`)
 	checkMatch("#name", TokenHash, "#name")
 	checkMatch("42''", TokenNumber, "42", TokenString, "''")
 	checkMatch("4.2", TokenNumber, "4.2")