Add setting to customise delimiters

engine.go

@@ -97,3 +97,11 @@ func (e *Engine) ParseAndRenderString(source string, b Bindings) (string, Source
 	}
 	return string(bs), nil
 }
+
+// Delims sets the action delimiters to the specified strings, to be used in subsequent calls to
+// ParseTemplate, ParseTemplateLocation, ParseAndRender, or ParseAndRenderString. An empty delimiter
+// stands for the corresponding default: objectLeft = {{, objectRight = }}, tagLeft = {% , tagRight = %}
+func (e *Engine) Delims(objectLeft, objectRight, tagLeft, tagRight string) *Engine {
+	e.cfg.Delims = []string{objectLeft, objectRight, tagLeft, tagRight}
+	return e
+}

parser/config.go

@@ -6,6 +6,7 @@ import "github.com/osteele/liquid/expressions"
 type Config struct {
 	expressions.Config
 	Grammar Grammar
+	Delims  []string
 }
 
 // NewConfig creates a parser Config.

parser/parser.go

@@ -10,7 +10,7 @@ import (
 
 // Parse parses a source template. It returns an AST root, that can be compiled and evaluated.
 func (c Config) Parse(source string, loc SourceLoc) (ASTNode, Error) {
-	tokens := Scan(source, loc)
+	tokens := Scan(source, loc, c.Delims)
 	return c.parseTokens(tokens)
 }
 

parser/scanner.go

@@ -1,14 +1,20 @@
 package parser
 
 import (
+	"fmt"
 	"regexp"
 	"strings"
 )
 
-var tokenMatcher = regexp.MustCompile(`{{-?\s*(.+?)\s*-?}}|{%-?\s*(\w+)(?:\s+((?:[^%]|%[^}])+?))?\s*-?%}`)
-
 // Scan breaks a string into a sequence of Tokens.
-func Scan(data string, loc SourceLoc) (tokens []Token) {
+func Scan(data string, loc SourceLoc, delims []string) (tokens []Token) {
+
+	// Apply defaults
+	if len(delims) != 4 {
+		delims = []string{"{{", "}}", "{%", "%}"}
+	}
+	tokenMatcher := formTokenMatcher(delims)
+
 	// TODO error on unterminated {{ and {%
 	// TODO probably an error when a tag contains a {{ or {%, at least outside of a string
 	p, pe := 0, len(data)
@@ -19,8 +25,8 @@ func Scan(data string, loc SourceLoc) (tokens []Token) {
 			loc.LineNo += strings.Count(data[p:ts], "\n")
 		}
 		source := data[ts:te]
-		switch data[ts+1] {
-		case '{':
+		switch {
+		case data[ts:ts+len(delims[0])] == delims[0]:
 			tok := Token{
 				Type:      ObjTokenType,
 				SourceLoc: loc,
@@ -30,7 +36,7 @@ func Scan(data string, loc SourceLoc) (tokens []Token) {
 				TrimRight: source[len(source)-3] == '-',
 			}
 			tokens = append(tokens, tok)
-		case '%':
+		case data[ts:ts+len(delims[2])] == delims[2]:
 			tok := Token{
 				Type:      TagTokenType,
 				SourceLoc: loc,
@@ -52,3 +58,28 @@ func Scan(data string, loc SourceLoc) (tokens []Token) {
 	}
 	return tokens
 }
+
+func formTokenMatcher(delims []string) *regexp.Regexp {
+	// On ending a tag we need to exclude anything that appears to be ending a tag that's nested
+	// inside the tag. We form the exlusion expression here.
+	// For example, if delims is default the exclusion expression is "[^%]|%[^}]".
+	// If tagRight is "TAG!RIGHT" then expression is
+	// [^T]|T[^A]|TA[^G]|TAG[^!]|TAG![^R]|TAG!R[^I]|TAG!RI[^G]|TAG!RIG[^H]|TAG!RIGH[^T]
+	exclusion := []string{}
+	for idx, val := range delims[3] {
+		exclusion = append(exclusion, "[^"+string(val)+"]")
+		if idx > 0 {
+			exclusion[idx] = delims[3][0:idx] + exclusion[idx]
+		}
+	}
+
+	tokenMatcher := regexp.MustCompile(
+		fmt.Sprintf(`%s-?\s*(.+?)\s*-?%s|%s-?\s*(\w+)(?:\s+((?:%v)+?))?\s*-?%s`,
+			// QuoteMeta will escape any of these that are regex commands
+			regexp.QuoteMeta(delims[0]), regexp.QuoteMeta(delims[1]),
+			regexp.QuoteMeta(delims[2]), strings.Join(exclusion, "|"), regexp.QuoteMeta(delims[3]),
+		),
+	)
+
+	return tokenMatcher
+}

parser/scanner_test.go

@@ -23,7 +23,7 @@ var scannerCountTests = []struct {
 }
 
 func TestScan(t *testing.T) {
-	scan := func(src string) []Token { return Scan(src, SourceLoc{}) }
+	scan := func(src string) []Token { return Scan(src, SourceLoc{}, nil) }
 	tokens := scan("12")
 	require.NotNil(t, tokens)
 	require.Len(t, tokens, 1)
@@ -69,7 +69,7 @@ func TestScan(t *testing.T) {
 
 func TestScan_ws(t *testing.T) {
 	// whitespace control
-	scan := func(src string) []Token { return Scan(src, SourceLoc{}) }
+	scan := func(src string) []Token { return Scan(src, SourceLoc{}, nil) }
 
 	wsTests := []struct {
 		in, expect  string
@@ -98,3 +98,65 @@ func TestScan_ws(t *testing.T) {
 		})
 	}
 }
+
+var scannerCountTestsDelims = []struct {
+	in  string
+	len int
+}{
+	{`TAG*LEFT tag arg TAG!RIGHT`, 1},
+	{`TAG*LEFT tag arg TAG!RIGHTTAG*LEFT tag TAG!RIGHT`, 2},
+	{`TAG*LEFT tag arg TAG!RIGHTTAG*LEFT tag arg TAG!RIGHTTAG*LEFT tag TAG!RIGHT`, 3},
+	{`TAG*LEFT tag TAG!RIGHTTAG*LEFT tag TAG!RIGHT`, 2},
+	{`TAG*LEFT tag arg TAG!RIGHTTAG*LEFT tag arg TAG!RIGHTTAG*LEFT tag TAG!RIGHTTAG*LEFT tag TAG!RIGHT`, 4},
+	{`OBJECT@LEFT expr OBJECT#RIGHT`, 1},
+	{`OBJECT@LEFT expr arg OBJECT#RIGHT`, 1},
+	{`OBJECT@LEFT expr OBJECT#RIGHTOBJECT@LEFT expr OBJECT#RIGHT`, 2},
+	{`OBJECT@LEFT expr arg OBJECT#RIGHTOBJECT@LEFT expr arg OBJECT#RIGHT`, 2},
+}
+
+func TestScan_delims(t *testing.T) {
+	scan := func(src string) []Token {
+		return Scan(src, SourceLoc{}, []string{"OBJECT@LEFT", "OBJECT#RIGHT", "TAG*LEFT", "TAG!RIGHT"})
+	}
+	tokens := scan("12")
+	require.NotNil(t, tokens)
+	require.Len(t, tokens, 1)
+	require.Equal(t, TextTokenType, tokens[0].Type)
+	require.Equal(t, "12", tokens[0].Source)
+
+	tokens = scan("OBJECT@LEFTobjOBJECT#RIGHT")
+	require.NotNil(t, tokens)
+	require.Len(t, tokens, 1)
+	require.Equal(t, ObjTokenType, tokens[0].Type)
+	require.Equal(t, "obj", tokens[0].Args)
+
+	tokens = scan("OBJECT@LEFT obj OBJECT#RIGHT")
+	require.NotNil(t, tokens)
+	require.Len(t, tokens, 1)
+	require.Equal(t, ObjTokenType, tokens[0].Type)
+	require.Equal(t, "obj", tokens[0].Args)
+
+	tokens = scan("TAG*LEFTtag argsTAG!RIGHT")
+	require.NotNil(t, tokens)
+	require.Len(t, tokens, 1)
+	require.Equal(t, TagTokenType, tokens[0].Type)
+	require.Equal(t, "tag", tokens[0].Name)
+	require.Equal(t, "args", tokens[0].Args)
+
+	tokens = scan("TAG*LEFT tag args TAG!RIGHT")
+	require.NotNil(t, tokens)
+	require.Len(t, tokens, 1)
+	require.Equal(t, TagTokenType, tokens[0].Type)
+	require.Equal(t, "tag", tokens[0].Name)
+	require.Equal(t, "args", tokens[0].Args)
+
+	tokens = scan("preTAG*LEFT tag args TAG!RIGHTmidOBJECT@LEFT object OBJECT#RIGHTpost")
+	require.Equal(t, `[TextTokenType{"pre"} TagTokenType{Tag:"tag", Args:"args"} TextTokenType{"mid"} ObjTokenType{"object"} TextTokenType{"post"}]`, fmt.Sprint(tokens))
+
+	for i, test := range scannerCountTestsDelims {
+		t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) {
+			tokens := scan(test.in)
+			require.Len(t, tokens, test.len)
+		})
+	}
+}