Start optimizations with ReHighlight and add python files

highlighter.go

@@ -18,9 +18,17 @@ func combineLineMatch(src, dst LineMatch) LineMatch {
 	return dst
 }
 
+type State *Region
+
+type LineStates interface {
+	Lines() [][]byte
+	State(lineN int) State
+	SetState(lineN int, s State)
+}
+
 type Highlighter struct {
-	states [][2]*Region
-	def    *Def
+	endRegions []*Region
+	def        *Def
 }
 
 func NewHighlighter(def *Def) *Highlighter {
@@ -66,7 +74,7 @@ func (h *Highlighter) highlightRegion(start int, canMatchEnd bool, lineNum int,
 
 	if len(line) == 0 {
 		if canMatchEnd {
-			h.states[lineNum][1] = region
+			h.endRegions[lineNum] = region
 		}
 
 		return highlights
@@ -107,7 +115,7 @@ func (h *Highlighter) highlightRegion(start int, canMatchEnd bool, lineNum int,
 	}
 
 	if canMatchEnd {
-		h.states[lineNum][1] = region
+		h.endRegions[lineNum] = region
 	}
 
 	return highlights
@@ -117,7 +125,7 @@ func (h *Highlighter) highlightEmptyRegion(start int, canMatchEnd bool, lineNum
 	highlights := make(LineMatch)
 	if len(line) == 0 {
 		if canMatchEnd {
-			h.states[lineNum][1] = nil
+			h.endRegions[lineNum] = nil
 		}
 		return highlights
 	}
@@ -143,34 +151,55 @@ func (h *Highlighter) highlightEmptyRegion(start int, canMatchEnd bool, lineNum
 	}
 
 	if canMatchEnd {
-		h.states[lineNum][1] = nil
+		h.endRegions[lineNum] = nil
 	}
 
 	return highlights
 }
 
-func (h *Highlighter) Highlight(input string, startline int) []LineMatch {
+func (h *Highlighter) Highlight(input string) []LineMatch {
 	lines := strings.Split(input, "\n")
 	var lineMatches []LineMatch
 
-	lastStates := h.states
-	h.states = make([][2]*Region, len(lines))
+	h.endRegions = make([]*Region, len(lines))
 
-	optimize := len(lastStates) == len(h.states)
-
-	for i := startline; i < len(lines); i++ {
+	for i := 0; i < len(lines); i++ {
 		line := []byte(lines[i])
 
-		if i != 0 && optimize && h.states[i-1][1] == lastStates[i-1][1] {
-			break
-		}
-
-		if i == 0 || h.states[i-1][1] == nil {
+		if i == 0 || h.endRegions[i-1] == nil {
 			lineMatches = append(lineMatches, h.highlightEmptyRegion(0, true, i, line))
 		} else {
-			lineMatches = append(lineMatches, h.highlightRegion(0, true, i, line, h.states[i-1][1]))
+			lineMatches = append(lineMatches, h.highlightRegion(0, true, i, line, h.endRegions[i-1]))
 		}
 	}
 
 	return lineMatches
 }
+
+func (h *Highlighter) ReHighlight(input LineStates, startline int) []LineMatch {
+	lines := input.Lines()
+	var lineMatches []LineMatch
+
+	h.endRegions = make([]*Region, len(lines))
+
+	for i := startline; i < len(lines); i++ {
+		line := []byte(lines[i])
+
+		if i == 0 || h.endRegions[i-1] == nil {
+			lineMatches = append(lineMatches, h.highlightEmptyRegion(0, true, i, line))
+		} else {
+			lineMatches = append(lineMatches, h.highlightRegion(0, true, i, line, h.endRegions[i-1]))
+		}
+
+		curState := h.endRegions[i]
+		lastState := input.State(i)
+
+		if curState == lastState {
+			break
+		}
+
+		input.SetState(i, curState)
+	}
+
+	return lineMatches
+}

syntax_files/python2.yaml

@@ -0,0 +1,57 @@
+filetype: python
+
+detect:
+    filename: "\\.py$"
+    header: "^#!.*/(env +)?python( |$)"
+
+rules:
+
+    # built-in objects
+    - constant: "\\b(None|self|True|False)\\b"
+      # built-in attributes
+    - constant: "\\b(__bases__|__builtin__|__class__|__debug__|__dict__|__doc__|__file__|__members__|__methods__|__name__|__self__)\\b"
+      # built-in functions
+    - identifier: "\\b(abs|apply|callable|chr|cmp|compile|delattr|dir|divmod|eval|exec|execfile|filter|format|getattr|globals|hasattr|hash|help|hex|id|input|intern|isinstance|issubclass|len|locals|max|min|next|oct|open|ord|pow|range|raw_input|reduce|reload|repr|round|setattr|unichr|vars|zip|__import__)\\b"
+      # special method names
+    - identifier: "\\b(__abs__|__add__|__and__|__call__|__cmp__|__coerce__|__complex__|__concat__|__contains__|__del__|__delattr__|__delitem__|__dict__|__delslice__|__div__|__divmod__|__float__|__getattr__|__getitem__|__getslice__|__hash__|__hex__|__init__|__int__|__inv__|__invert__|__len__|__long__|__lshift__|__mod__|__mul__|__neg__|__nonzero__|__oct__|__or__|__pos__|__pow__|__radd__|__rand__|__rcmp__|__rdiv__|__rdivmod__|__repeat__|__repr__|__rlshift__|__rmod__|__rmul__|__ror__|__rpow__|__rrshift__|__rshift__|__rsub__|__rxor__|__setattr__|__setitem__|__setslice__|__str__|__sub__|__xor__)\\b"
+      # types
+    - type: "\\b(basestring|bool|buffer|bytearray|bytes|classmethod|complex|dict|enumerate|file|float|frozenset|int|list|long|map|memoryview|object|property|reversed|set|slice|staticmethod|str|super|tuple|type|unicode|xrange)\\b"
+      # definitions
+    - identifier: "def [a-zA-Z_0-9]+"
+      # keywords
+    - statement: "\\b(and|as|assert|break|class|continue|def|del|elif|else|except|finally|for|from|global|if|import|in|is|lambda|not|or|pass|print|raise|return|try|while|with|yield)\\b"
+      # decorators
+    - brightgreen: "@.*[(]"
+      # operators
+    - statement: "[.:;,+*|=!\\%@]" "<" ">" "/" "-" "&"
+      # parentheses
+    - statement: "[(){}]" "\\[" "\\]"
+      # numbers
+    - constant.number: "\\b[0-9]+\\b"
+
+    - constant.string:
+        start: "\""
+        end: "\""
+        rules:
+            - constant.specialChar: "\\\\."
+
+    - constant.string:
+        start: "'"
+        end: "'"
+        rules:
+            - constant.specialChar: "\\\\."
+
+    - comment:
+        start: "#"
+        end: "$"
+        rules: []
+
+    - comment:
+        start: "\"\"\""
+        end: "\"\"\""
+        rules: []
+
+    - comment:
+        start: "'''"
+        end: "'''"
+        rules: []

syntax_files/python3.yaml

@@ -0,0 +1,56 @@
+filename: python3
+
+detect:
+    filename: "\\.py3$"
+    header: "^#!.*/(env +)?python3$"
+
+rules:
+    # built-in objects
+    - constant: "\\b(None|self|True|False)\\b"
+      # built-in attributes
+    - constant: "\\b(__bases__|__builtin__|__class__|__debug__|__dict__|__doc__|__file__|__members__|__methods__|__name__|__self__)\\b"
+      # built-in functions
+    - identifier: "\\b(abs|all|any|ascii|bin|callable|chr|compile|delattr|dir|divmod|eval|exec|format|getattr|globals|hasattr|hash|help|hex|id|input|isinstance|issubclass|iter|len|locals|max|min|next|oct|open|ord|pow|print|repr|round|setattr|sorted|sum|vars|__import__)\\b"
+      # special method names
+    - identifier: "\\b(__abs__|__add__|__and__|__call__|__cmp__|__coerce__|__complex__|__concat__|__contains__|__del__|__delattr__|__delitem__|__delslice__|__div__|__divmod__|__float__|__getattr__|__getitem__|__getslice__|__hash__|__hex__|__init__|__int__|__inv__|__invert__|__len__|__dict__|__long__|__lshift__|__mod__|__mul__|__neg__|__next__|__nonzero__|__oct__|__or__|__pos__|__pow__|__radd__|__rand__|__rcmp__|__rdiv__|__rdivmod__|__repeat__|__repr__|__rlshift__|__rmod__|__rmul__|__ror__|__rpow__|__rrshift__|__rshift__|__rsub__|__rxor__|__setattr__|__setitem__|__setslice__|__str__|__sub__|__xor__)\\b"
+      # types
+    - type: "\\b(bool|bytearray|bytes|classmethod|complex|dict|enumerate|filter|float|frozenset|int|list|map|memoryview|object|property|range|reversed|set|slice|staticmethod|str|super|tuple|type|zip)\\b"
+      # definitions
+    - identifier: "def [a-zA-Z_0-9]+" 
+      # keywords
+    - statement: "\\b(and|as|assert|break|class|continue|def|del|elif|else|except|finally|for|from|global|if|import|in|is|lambda|nonlocal|not|or|pass|raise|return|try|while|with|yield)\\b" 
+      # decorators
+    - brightgreen: "@.*[(]"
+      # operators
+    - statement: "[.:;,+*|=!\\%@]" "<" ">" "/" "-" "&"
+      # parentheses
+    - statement: "[(){}]" "\\[" "\\]"
+      # numbers
+    - constant.number: "\\b[0-9]+\\b"
+
+    - constant.string:
+        start: "\""
+        end: "\""
+        rules:
+            - constant.specialChar: "\\\\."
+
+    - constant.string:
+        start: "'"
+        end: "'"
+        rules:
+            - constant.specialChar: "\\\\."
+
+    - comment:
+        start: "#"
+        end: "$"
+        rules: []
+
+    - comment:
+        start: "\"\"\""
+        end: "\"\"\""
+        rules: []
+
+    - comment:
+        start: "'''"
+        end: "'''"
+        rules: []