From 004edc7daa3c63288cf8d5f035f6f23d2c858f18 Mon Sep 17 00:00:00 2001
From: Rainer Kottenhoff <rainer.kottenhoff@gmail.com>
Date: Fri, 29 Mar 2019 19:40:30 +0100
Subject: [PATCH] + upd: current Scintilla dev on GitHub mirror

---
 scintilla/cppcheck.suppress                   |  3 +
 scintilla/doc/ScintillaDoc.html               | 11 +++
 scintilla/doc/ScintillaHistory.html           | 10 ++
 scintilla/include/Scintilla.h                 |  6 +-
 scintilla/include/Scintilla.iface             | 10 +-
 scintilla/lexers/LexCIL.cxx                   |  1 -
 scintilla/lexers/LexCPP.cxx                   | 96 ++++++++++++-------
 scintilla/lexers/LexHaskell.cxx               |  1 +
 scintilla/lexers/LexNim.cxx                   |  1 -
 scintilla/lexlib/CharacterCategory.cxx        | 31 ++++++
 scintilla/lexlib/CharacterCategory.h          | 17 ++++
 .../scripts/GenerateCharacterCategory.py      | 20 +++-
 scintilla/src/Document.cxx                    | 10 +-
 scintilla/src/Document.h                      |  4 +
 scintilla/src/EditModel.cxx                   |  2 +
 scintilla/src/EditView.cxx                    |  1 +
 scintilla/src/Editor.cxx                      |  8 ++
 scintilla/src/MarginView.cxx                  |  1 +
 scintilla/src/PositionCache.cxx               |  1 +
 scintilla/src/ScintillaBase.cxx               |  1 +
 scintilla/win32/ScintillaWin.cxx              |  1 +
 src/Edit.c                                    |  2 +
 src/Encoding.c                                | 35 ++++++-
 src/Encoding.h                                |  3 +-
 src/SciCall.h                                 |  2 +-
 src/Version.h                                 |  2 +-
 26 files changed, 229 insertions(+), 51 deletions(-)

diff --git a/scintilla/cppcheck.suppress b/scintilla/cppcheck.suppress
index dc1f264e1..4bf51148f 100644
--- a/scintilla/cppcheck.suppress
+++ b/scintilla/cppcheck.suppress
@@ -71,6 +71,9 @@ unreadVariable:scintilla/lexers/LexVisualProlog.cxx
 // bp.itBracket not actually redundant as needed by return statements
 redundantAssignment:scintilla/lexers/LexCPP.cxx
 
+// safety initializations at start of GetCharacterExtents
+redundantAssignment:scintilla/gtk/ScintillaGTKAccessible.cxx
+
 // Suppress everything in catch.hpp as won't be changing
 *:scintilla/test/unit/catch.hpp
 // For now, suppress all test source files as, since Catch 2, cppcheck shows many warnings showing
diff --git a/scintilla/doc/ScintillaDoc.html b/scintilla/doc/ScintillaDoc.html
index 40bb8f430..ad030d235 100644
--- a/scintilla/doc/ScintillaDoc.html
+++ b/scintilla/doc/ScintillaDoc.html
@@ -2470,6 +2470,8 @@ struct Sci_TextToFind {
      <a class="message" href="#SCI_SETPUNCTUATIONCHARS">SCI_SETPUNCTUATIONCHARS(&lt;unused&gt;, const char *characters)</a><br />
      <a class="message" href="#SCI_GETPUNCTUATIONCHARS">SCI_GETPUNCTUATIONCHARS(&lt;unused&gt;, char *characters) &rarr; int</a><br />
      <a class="message" href="#SCI_SETCHARSDEFAULT">SCI_SETCHARSDEFAULT</a><br />
+     <a class="message" href="#SCI_SETCHARACTERCATEGORYOPTIMIZATION">SCI_SETCHARACTERCATEGORYOPTIMIZATION(int countCharacters)</a><br />
+     <a class="message" href="#SCI_GETCHARACTERCATEGORYOPTIMIZATION">SCI_GETCHARACTERCATEGORYOPTIMIZATION &rarr; int</a><br />
 
     <p><b id="SCI_WORDENDPOSITION">SCI_WORDENDPOSITION(int pos, bool onlyWordCharacters) &rarr; int</b><br />
      <b id="SCI_WORDSTARTPOSITION">SCI_WORDSTARTPOSITION(int pos, bool onlyWordCharacters) &rarr; int</b><br />
@@ -2597,6 +2599,15 @@ struct Sci_TextToFind {
      characters with codes less than 0x20, with word characters set to alphanumeric and '_'.
     </p>
 
+    <p><b id="SCI_SETCHARACTERCATEGORYOPTIMIZATION">SCI_SETCHARACTERCATEGORYOPTIMIZATION(int countCharacters)</b><br />
+    <b id="SCI_GETCHARACTERCATEGORYOPTIMIZATION">SCI_GETCHARACTERCATEGORYOPTIMIZATION &rarr; int</b><br />
+      Optimize speed of character category features like determining whether a character is a space or number at the expense of memory.
+      Mostly used for Unicode documents.
+      The countCharacters parameter determines how many character starting from 0 are added to a look-up table with one byte used for each character.
+      It is reasonable to cover the set of characters likely to be used in a document so 0x100 for simple Roman text,
+      0x1000 to cover most simple alphabets, 0x10000 to cover most of East Asian languages, and 0x110000 to cover all possible characters.
+    </p>
+
     <p>Word keyboard commands are:</p>
     <ul>
     <li class="message" id="SCI_WORDLEFT">SCI_WORDLEFT</li>
diff --git a/scintilla/doc/ScintillaHistory.html b/scintilla/doc/ScintillaHistory.html
index 381dc4ecd..8e8d734dd 100644
--- a/scintilla/doc/ScintillaHistory.html
+++ b/scintilla/doc/ScintillaHistory.html
@@ -571,6 +571,16 @@
 	Implement high-priority idle on Win32 to make redraw smoother and more efficient.
 	</li>
  	<li>
+	Add SCI_SETCHARACTERCATEGORYOPTIMIZATION API to optimize speed
+	of character category features like determining whether a character is a space or number
+	at the expense of memory.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1259/">Feature #1259</a>.
+	</li>
+ 	<li>
+	Fix flicker when inserting primary selection on GTK.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2087/">Bug #2087</a>.
+	</li>
+ 	<li>
 	Avoid potential long hangs with idle styling for huge documents on Cocoa and GTK.
 	</li>
     <ul>
diff --git a/scintilla/include/Scintilla.h b/scintilla/include/Scintilla.h
index 1d119370d..f1e01c710 100644
--- a/scintilla/include/Scintilla.h
+++ b/scintilla/include/Scintilla.h
@@ -106,8 +106,8 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SCI_GETIMEINTERACTION 2678
 #define SCI_SETIMEINTERACTION 2679
 // >>>>>>>>>>>>>>>   BEG NON STD SCI PATCH   >>>>>>>>>>>>>>>
-#define SCI_ISIMEOPEN 2719
-#define SCI_ISIMEMODECJK 2720
+#define SCI_ISIMEOPEN 2722
+#define SCI_ISIMEMODECJK 2723
 // <<<<<<<<<<<<<<<   END NON STD SCI PATCH   <<<<<<<<<<<<<<<
 #define MARKER_MAX 31
 #define SC_MARK_CIRCLE 0
@@ -273,6 +273,8 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SCI_SETCARETPERIOD 2076
 #define SCI_SETWORDCHARS 2077
 #define SCI_GETWORDCHARS 2646
+#define SCI_SETCHARACTERCATEGORYOPTIMIZATION 2720
+#define SCI_GETCHARACTERCATEGORYOPTIMIZATION 2721
 #define SCI_BEGINUNDOACTION 2078
 #define SCI_ENDUNDOACTION 2079
 #define INDIC_PLAIN 0
diff --git a/scintilla/include/Scintilla.iface b/scintilla/include/Scintilla.iface
index e19f3cfb2..9ffd4157c 100644
--- a/scintilla/include/Scintilla.iface
+++ b/scintilla/include/Scintilla.iface
@@ -267,8 +267,8 @@ get int GetIMEInteraction=2678(,)
 set void SetIMEInteraction=2679(int imeInteraction,)
 
 # >>>>>>>>>>>>>>>   BEG NON STD SCI PATCH   >>>>>>>>>>>>>>>
-get bool IsIMEOpen=2719(,)
-get bool IsIMEModeCJK=2720(,)
+get bool IsIMEOpen=2722(,)
+get bool IsIMEModeCJK=2723(,)
 # <<<<<<<<<<<<<<<   END NON STD SCI PATCH   <<<<<<<<<<<<<<<
 
 enu MarkerSymbol=SC_MARK_
@@ -615,6 +615,12 @@ set void SetWordChars=2077(, string characters)
 # Returns the number of characters
 get int GetWordChars=2646(, stringresult characters)
 
+# Set the number of characters to have directly indexed categories
+set void SetCharacterCategoryOptimization=2720(int countCharacters,)
+
+# Get the number of characters to have directly indexed categories
+get int GetCharacterCategoryOptimization=2721(,)
+
 # Start a sequence of actions that is undone and redone as a unit.
 # May be nested.
 fun void BeginUndoAction=2078(,)
diff --git a/scintilla/lexers/LexCIL.cxx b/scintilla/lexers/LexCIL.cxx
index 6d5307666..6f498315a 100644
--- a/scintilla/lexers/LexCIL.cxx
+++ b/scintilla/lexers/LexCIL.cxx
@@ -28,7 +28,6 @@
 #include "Accessor.h"
 #include "StyleContext.h"
 #include "CharacterSet.h"
-#include "CharacterCategory.h"
 #include "LexerModule.h"
 #include "OptionSet.h"
 #include "DefaultLexer.h"
diff --git a/scintilla/lexers/LexCPP.cxx b/scintilla/lexers/LexCPP.cxx
index 49921fd8e..3b8bbfa16 100644
--- a/scintilla/lexers/LexCPP.cxx
+++ b/scintilla/lexers/LexCPP.cxx
@@ -245,12 +245,24 @@ struct PPDefinition {
 	}
 };
 
+const int inactiveFlag = 0x40;
+
 class LinePPState {
-	int state;
-	int ifTaken;
-	int level;
+	// Track the state of preprocessor conditionals to allow showing active and inactive
+	// code in different styles.
+	// Only works up to 31 levels of conditional nesting.
+
+	// state is a bit mask with 1 bit per level
+	// bit is 1 for level if section inactive, so any bits set = inactive style
+	int state = 0;
+	// ifTaken is a bit mask with 1 bit per level
+	// bit is 1 for level if some branch at this level has been taken
+	int ifTaken = 0;
+	// level is the nesting level of #if constructs
+	int level = -1;
+	static const int maximumNestingLevel = 31;
 	bool ValidLevel() const noexcept {
-		return level >= 0 && level < 32;
+		return level >= 0 && level < maximumNestingLevel;
 	}
 	int maskLevel() const noexcept {
 		if (level >= 0) {
@@ -260,11 +272,17 @@ class LinePPState {
 		}
 	}
 public:
-	LinePPState() noexcept : state(0), ifTaken(0), level(-1) {
+	LinePPState() noexcept {
+	}
+	bool IsActive() const noexcept {
+		return state == 0;
 	}
 	bool IsInactive() const noexcept {
 		return state != 0;
 	}
+	int ActiveState() const noexcept {
+		return state ? inactiveFlag : 0;
+	}
 	bool CurrentIfTaken() const noexcept {
 		return (ifTaken & maskLevel()) != 0;
 	}
@@ -521,7 +539,6 @@ class LexerCPP : public ILexer4 {
 	OptionSetCPP osCPP;
 	EscapeSequence escapeSeq;
 	SparseState<std::string> rawStringTerminators;
-	enum { activeFlag = 0x40 };
 	enum { ssIdentifier, ssDocKeyword };
 	SubStyles subStyles;
 	std::string returnBuffer;
@@ -534,7 +551,7 @@ public:
 		setMultOp(CharacterSet::setNone, "*/%"),
 		setRelOp(CharacterSet::setNone, "=!<>"),
 		setLogicalOp(CharacterSet::setNone, "|&"),
-		subStyles(styleSubable, 0x80, 0x40, activeFlag) {
+		subStyles(styleSubable, 0x80, 0x40, inactiveFlag) {
 	}
 	// Deleted so LexerCPP objects can not be copied.
 	LexerCPP(const LexerCPP &) = delete;
@@ -585,8 +602,8 @@ public:
 	}
 	int SCI_METHOD StyleFromSubStyle(int subStyle) override {
 		const int styleBase = subStyles.BaseStyle(MaskActive(subStyle));
-		const int active = subStyle & activeFlag;
-		return styleBase | active;
+		const int inactive = subStyle & inactiveFlag;
+		return styleBase | inactive;
 	}
 	int SCI_METHOD PrimaryStyleFromStyle(int style) noexcept override {
 		return MaskActive(style);
@@ -598,7 +615,7 @@ public:
 		subStyles.SetIdentifiers(style, identifiers);
 	}
 	int SCI_METHOD DistanceToSecondaryStyles() noexcept override {
-		return activeFlag;
+		return inactiveFlag;
 	}
 	const char * SCI_METHOD GetSubStyleBases() noexcept override {
 		return styleSubable;
@@ -606,7 +623,7 @@ public:
 	int SCI_METHOD NamedStyles() override {
 		return std::max(subStyles.LastAllocated() + 1,
 			sizeLexicalClasses) +
-			activeFlag;
+			inactiveFlag;
 	}
 	const char * SCI_METHOD NameOfStyle(int style) override {
 		if (style >= NamedStyles())
@@ -624,11 +641,11 @@ public:
 		if (firstSubStyle >= 0) {
 			const int lastSubStyle = subStyles.LastAllocated();
 			if (((style >= firstSubStyle) && (style <= (lastSubStyle))) ||
-				((style >= firstSubStyle + activeFlag) && (style <= (lastSubStyle + activeFlag)))) {
+				((style >= firstSubStyle + inactiveFlag) && (style <= (lastSubStyle + inactiveFlag)))) {
 				int styleActive = style;
 				if (style > lastSubStyle) {
 					returnBuffer = "inactive ";
-					styleActive -= activeFlag;
+					styleActive -= inactiveFlag;
 				}
 				const int styleMain = StyleFromSubStyle(styleActive);
 				returnBuffer += lexicalClasses[styleMain].tags;
@@ -637,9 +654,9 @@ public:
 		}
 		if (style < sizeLexicalClasses)
 			return lexicalClasses[style].tags;
-		if (style >= activeFlag) {
+		if (style >= inactiveFlag) {
 			returnBuffer = "inactive ";
-			const int styleActive = style - activeFlag;
+			const int styleActive = style - inactiveFlag;
 			if (styleActive < sizeLexicalClasses)
 				returnBuffer += lexicalClasses[styleActive].tags;
 			else
@@ -664,7 +681,7 @@ public:
 		return new LexerCPP(false);
 	}
 	constexpr static int MaskActive(int style) noexcept {
-		return style & ~activeFlag;
+		return style & ~inactiveFlag;
 	}
 	void EvaluateTokens(std::vector<std::string> &tokens, const SymbolTable &preprocessorDefinitions);
 	std::vector<std::string> Tokenize(const std::string &expr) const;
@@ -822,7 +839,7 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 	std::string rawStringTerminator = rawStringTerminators.ValueAt(lineCurrent-1);
 	SparseState<std::string> rawSTNew(lineCurrent);
 
-	int activitySet = preproc.IsInactive() ? activeFlag : 0;
+	int activitySet = preproc.ActiveState();
 
 	const WordClassifier &classifierIdentifiers = subStyles.Classifier(SCE_C_IDENTIFIER);
 	const WordClassifier &classifierDocKeyWords = subStyles.Classifier(SCE_C_COMMENTDOCKEYWORD);
@@ -849,7 +866,7 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 			isIncludePreprocessor = false;
 			inRERange = false;
 			if (preproc.IsInactive()) {
-				activitySet = activeFlag;
+				activitySet = inactiveFlag;
 				sc.SetState(sc.state | activitySet);
 			}
 		}
@@ -1274,6 +1291,8 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 					isIncludePreprocessor = true;
 				} else {
 					if (options.trackPreprocessor) {
+						// If #if is nested too deeply (>31 levels) the active/inactive appearance
+						// will stop reflecting the code.
 						if (sc.Match("ifdef") || sc.Match("ifndef")) {
 							const bool isIfDef = sc.Match("ifdef");
 							const int startRest = isIfDef ? 5 : 6;
@@ -1285,41 +1304,52 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 							const bool ifGood = EvaluateExpression(restOfLine, preprocessorDefinitions);
 							preproc.StartSection(ifGood);
 						} else if (sc.Match("else")) {
+							// #else is shown as active if either preceding or following section is active
+							// as that means that it contributed to the result.
 							if (!preproc.CurrentIfTaken()) {
+								// Inactive, may become active if parent scope active
+								assert(sc.state == (SCE_C_PREPROCESSOR|inactiveFlag));
 								preproc.InvertCurrentLevel();
-								activitySet = preproc.IsInactive() ? activeFlag : 0;
+								activitySet = preproc.ActiveState();
+								// If following is active then show "else" as active
 								if (!activitySet)
-									sc.ChangeState(SCE_C_PREPROCESSOR|activitySet);
-							} else if (!preproc.IsInactive()) {
+									sc.ChangeState(SCE_C_PREPROCESSOR);
+							} else if (preproc.IsActive()) {
+								// Active -> inactive
+								assert(sc.state == SCE_C_PREPROCESSOR);
 								preproc.InvertCurrentLevel();
-								activitySet = preproc.IsInactive() ? activeFlag : 0;
-								if (!activitySet)
-									sc.ChangeState(SCE_C_PREPROCESSOR|activitySet);
+								activitySet = preproc.ActiveState();
+								// Continue to show "else" as active as it ends active section.
 							}
 						} else if (sc.Match("elif")) {
 							// Ensure only one chosen out of #if .. #elif .. #elif .. #else .. #endif
+							// #elif is shown as active if either preceding or following section is active
+							// as that means that it contributed to the result.
 							if (!preproc.CurrentIfTaken()) {
+								// Inactive, if expression true then may become active if parent scope active
+								assert(sc.state == (SCE_C_PREPROCESSOR|inactiveFlag));
 								// Similar to #if
 								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 4, true);
 								const bool ifGood = EvaluateExpression(restOfLine, preprocessorDefinitions);
 								if (ifGood) {
 									preproc.InvertCurrentLevel();
-									activitySet = preproc.IsInactive() ? activeFlag : 0;
+									activitySet = preproc.ActiveState();
 									if (!activitySet)
-										sc.ChangeState(SCE_C_PREPROCESSOR|activitySet);
+										sc.ChangeState(SCE_C_PREPROCESSOR);
 								}
-							} else if (!preproc.IsInactive()) {
+							} else if (preproc.IsActive()) {
+								// Active -> inactive
+								assert(sc.state == SCE_C_PREPROCESSOR);
 								preproc.InvertCurrentLevel();
-								activitySet = preproc.IsInactive() ? activeFlag : 0;
-								if (!activitySet)
-									sc.ChangeState(SCE_C_PREPROCESSOR|activitySet);
+								activitySet = preproc.ActiveState();
+								// Continue to show "elif" as active as it ends active section.
 							}
 						} else if (sc.Match("endif")) {
 							preproc.EndSection();
-							activitySet = preproc.IsInactive() ? activeFlag : 0;
+							activitySet = preproc.ActiveState();
 							sc.ChangeState(SCE_C_PREPROCESSOR|activitySet);
 						} else if (sc.Match("define")) {
-							if (options.updatePreprocessor && !preproc.IsInactive()) {
+							if (options.updatePreprocessor && preproc.IsActive()) {
 								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
 								size_t startName = 0;
 								while ((startName < restOfLine.length()) && IsSpaceOrTab(restOfLine[startName]))
@@ -1357,7 +1387,7 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 								}
 							}
 						} else if (sc.Match("undef")) {
-							if (options.updatePreprocessor && !preproc.IsInactive()) {
+							if (options.updatePreprocessor && preproc.IsActive()) {
 								const std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 5, false);
 								std::vector<std::string> tokens = Tokenize(restOfLine);
 								if (tokens.size() >= 1) {
diff --git a/scintilla/lexers/LexHaskell.cxx b/scintilla/lexers/LexHaskell.cxx
index ee8c87e39..8ed53d772 100644
--- a/scintilla/lexers/LexHaskell.cxx
+++ b/scintilla/lexers/LexHaskell.cxx
@@ -26,6 +26,7 @@
 #include <ctype.h>
 
 #include <string>
+#include <vector>
 #include <map>
 
 #include "ILexer.h"
diff --git a/scintilla/lexers/LexNim.cxx b/scintilla/lexers/LexNim.cxx
index 45a85469c..48c85d8ba 100644
--- a/scintilla/lexers/LexNim.cxx
+++ b/scintilla/lexers/LexNim.cxx
@@ -28,7 +28,6 @@
 #include "Accessor.h"
 #include "StyleContext.h"
 #include "CharacterSet.h"
-#include "CharacterCategory.h"
 #include "LexerModule.h"
 #include "OptionSet.h"
 #include "DefaultLexer.h"
diff --git a/scintilla/lexlib/CharacterCategory.cxx b/scintilla/lexlib/CharacterCategory.cxx
index bc2fa2336..ca76325df 100644
--- a/scintilla/lexlib/CharacterCategory.cxx
+++ b/scintilla/lexlib/CharacterCategory.cxx
@@ -7,6 +7,7 @@
 // Copyright 2013 by Neil Hodgson <neilh@scintilla.org>
 // The License.txt file describes the conditions under which this software may be distributed.
 
+#include <vector>
 #include <algorithm>
 #include <iterator>
 
@@ -3790,6 +3791,7 @@ const int catRanges[] = {
 33554397,
 33554460,
 35651549,
+35651613,
 //--Autogenerated -- end of section automatically generated
 };
 
@@ -3963,4 +3965,33 @@ bool IsXidContinue(int character) {
 	}
 }
 
+CharacterCategoryMap::CharacterCategoryMap() noexcept {
+	Optimize(256);
+}
+
+int CharacterCategoryMap::Size() const noexcept {
+	return static_cast<int>(dense.size());
+}
+
+void CharacterCategoryMap::Optimize(int countCharacters) {
+	const int characters = std::clamp(countCharacters, 256, maxUnicode + 1);
+	dense.resize(characters);
+
+	int end = 0;
+	int index = 0;
+	int current = catRanges[index];
+	++index;
+	do {
+		const int next = catRanges[index];
+		const unsigned char category = current & maskCategory;
+		current >>= 5;
+		end = std::min(characters, next >> 5);
+		while (current < end) {
+			dense[current++] = category;
+		}
+		current = next;
+		++index;
+	} while (characters > end);
+}
+
 }
diff --git a/scintilla/lexlib/CharacterCategory.h b/scintilla/lexlib/CharacterCategory.h
index 767d79670..d1ac39152 100644
--- a/scintilla/lexlib/CharacterCategory.h
+++ b/scintilla/lexlib/CharacterCategory.h
@@ -28,6 +28,23 @@ bool IsIdContinue(int character);
 bool IsXidStart(int character);
 bool IsXidContinue(int character);
 
+class CharacterCategoryMap {
+private:
+	std::vector<unsigned char> dense;
+public:
+	CharacterCategoryMap() noexcept;
+	CharacterCategory CategoryFor(int character) const {
+		if (static_cast<size_t>(character) < dense.size()) {
+			return static_cast<CharacterCategory>(dense[character]);
+		} else {
+			// binary search through ranges
+			return CategoriseCharacter(character);
+		}
+	}
+	int Size() const noexcept;
+	void Optimize(int countCharacters);
+};
+
 }
 
 #endif
diff --git a/scintilla/scripts/GenerateCharacterCategory.py b/scintilla/scripts/GenerateCharacterCategory.py
index 4596eec6a..ba6ac858d 100644
--- a/scintilla/scripts/GenerateCharacterCategory.py
+++ b/scintilla/scripts/GenerateCharacterCategory.py
@@ -17,17 +17,27 @@ def findCategories(filename):
 def updateCharacterCategory(filename):
     values = ["// Created with Python %s,  Unicode %s" % (
         platform.python_version(), unicodedata.unidata_version)]
-    category = unicodedata.category(chr(0))
+
     startRange = 0
+    category = unicodedata.category(chr(startRange))
+    table = []
     for ch in range(sys.maxunicode):
         uch = chr(ch)
-        if unicodedata.category(uch) != category:
+        current = unicodedata.category(uch)
+        if current != category:
             value = startRange * 32 + categories.index(category)
-            values.append("%d," % value)
-            category = unicodedata.category(uch)
+            table.append(value)
+            category = current
             startRange = ch
     value = startRange * 32 + categories.index(category)
-    values.append("%d," % value)
+    table.append(value)
+
+    # the sentinel value is used to simplify CharacterCategoryMap::Optimize()
+    category = 'Cn'
+    value = (sys.maxunicode + 1)*32 + categories.index(category)
+    table.append(value)
+
+    values.extend(["%d," % value for value in table])
 
     Regenerate(filename, "//", values)
 
diff --git a/scintilla/src/Document.cxx b/scintilla/src/Document.cxx
index 4df512b1d..72dae5f5b 100644
--- a/scintilla/src/Document.cxx
+++ b/scintilla/src/Document.cxx
@@ -1707,7 +1707,7 @@ CharClassify::cc Document::WordCharacterClass(unsigned int ch) const {
 	if (dbcsCodePage && (!UTF8IsAscii(ch))) {
 		if (SC_CP_UTF8 == dbcsCodePage) {
 			// Use hard coded Unicode class
-			const CharacterCategory cc = CategoriseCharacter(ch);
+			const CharacterCategory cc = charMap.CategoryFor(ch);
 			switch (cc) {
 
 				// Separator, Line/Paragraph
@@ -2166,6 +2166,14 @@ int Document::GetCharsOfClass(CharClassify::cc characterClass, unsigned char *bu
     return charClass.GetCharsOfClass(characterClass, buffer);
 }
 
+void Document::SetCharacterCategoryOptimization(int countCharacters) {
+	charMap.Optimize(countCharacters);
+}
+
+int Document::CharacterCategoryOptimization() const noexcept {
+	return charMap.Size();
+}
+
 void SCI_METHOD Document::StartStyling(Sci_Position position) {
 	endStyled = position;
 }
diff --git a/scintilla/src/Document.h b/scintilla/src/Document.h
index 96b1f8fd2..0c08b1b3c 100644
--- a/scintilla/src/Document.h
+++ b/scintilla/src/Document.h
@@ -10,6 +10,7 @@
 
 // >>>>>>>>>>>>>>>   BEG NON STD SCI PATCH   >>>>>>>>>>>>>>>
 #include "ILoader.h"
+#include "CharacterCategory.h"
 // <<<<<<<<<<<<<<<   END NON STD SCI PATCH   <<<<<<<<<<<<<<<
 
 namespace Scintilla {
@@ -234,6 +235,7 @@ private:
 	int refCount;
 	CellBuffer cb;
 	CharClassify charClass;
+	CharacterCategoryMap charMap;
 	std::unique_ptr<CaseFolder> pcf;
 	Sci::Position endStyled;
 	int styleClock;
@@ -448,6 +450,8 @@ public:
 	void SetDefaultCharClasses(bool includeWordClass);
 	void SetCharClasses(const unsigned char *chars, CharClassify::cc newCharClass);
 	int GetCharsOfClass(CharClassify::cc characterClass, unsigned char *buffer) const;
+	void SetCharacterCategoryOptimization(int countCharacters);
+	int CharacterCategoryOptimization() const noexcept;
 	void SCI_METHOD StartStyling(Sci_Position position) override;
 	bool SCI_METHOD SetStyleFor(Sci_Position length, char style) override;
 	bool SCI_METHOD SetStyles(Sci_Position length, const char *styles) override;
diff --git a/scintilla/src/EditModel.cxx b/scintilla/src/EditModel.cxx
index 70704e63d..f7a885c00 100644
--- a/scintilla/src/EditModel.cxx
+++ b/scintilla/src/EditModel.cxx
@@ -25,6 +25,8 @@
 #include "ILexer.h"
 #include "Scintilla.h"
 
+#include "CharacterCategory.h"
+
 #include "Position.h"
 #include "UniqueString.h"
 #include "SplitVector.h"
diff --git a/scintilla/src/EditView.cxx b/scintilla/src/EditView.cxx
index 9e8ca3474..ba5f102e0 100644
--- a/scintilla/src/EditView.cxx
+++ b/scintilla/src/EditView.cxx
@@ -30,6 +30,7 @@
 #include "Scintilla.h"
 
 #include "CharacterSet.h"
+#include "CharacterCategory.h"
 #include "Position.h"
 #include "IntegerRectangle.h"
 #include "UniqueString.h"
diff --git a/scintilla/src/Editor.cxx b/scintilla/src/Editor.cxx
index f3bf5babf..074def172 100644
--- a/scintilla/src/Editor.cxx
+++ b/scintilla/src/Editor.cxx
@@ -30,6 +30,7 @@
 #include "Scintilla.h"
 
 #include "CharacterSet.h"
+#include "CharacterCategory.h"
 #include "Position.h"
 #include "UniqueString.h"
 #include "SplitVector.h"
@@ -6254,6 +6255,13 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 		pdoc->SetDefaultCharClasses(true);
 		break;
 
+	case SCI_SETCHARACTERCATEGORYOPTIMIZATION:
+		pdoc->SetCharacterCategoryOptimization(static_cast<int>(wParam));
+		break;
+
+	case SCI_GETCHARACTERCATEGORYOPTIMIZATION:
+		return pdoc->CharacterCategoryOptimization();
+
 	case SCI_GETLENGTH:
 		return pdoc->Length();
 
diff --git a/scintilla/src/MarginView.cxx b/scintilla/src/MarginView.cxx
index 3f386455a..7a6b3a4a0 100644
--- a/scintilla/src/MarginView.cxx
+++ b/scintilla/src/MarginView.cxx
@@ -26,6 +26,7 @@
 #include "ILexer.h"
 #include "Scintilla.h"
 
+#include "CharacterCategory.h"
 #include "Position.h"
 #include "IntegerRectangle.h"
 #include "UniqueString.h"
diff --git a/scintilla/src/PositionCache.cxx b/scintilla/src/PositionCache.cxx
index 65c90dd1d..17bde4891 100644
--- a/scintilla/src/PositionCache.cxx
+++ b/scintilla/src/PositionCache.cxx
@@ -25,6 +25,7 @@
 #include "ILexer.h"
 #include "Scintilla.h"
 
+#include "CharacterCategory.h"
 #include "Position.h"
 #include "UniqueString.h"
 #include "SplitVector.h"
diff --git a/scintilla/src/ScintillaBase.cxx b/scintilla/src/ScintillaBase.cxx
index 75ca5b853..71e6fdb13 100644
--- a/scintilla/src/ScintillaBase.cxx
+++ b/scintilla/src/ScintillaBase.cxx
@@ -29,6 +29,7 @@
 #endif
 
 #include "PropSetSimple.h"
+#include "CharacterCategory.h"
 
 #ifdef SCI_LEXER
 #include "LexerModule.h"
diff --git a/scintilla/win32/ScintillaWin.cxx b/scintilla/win32/ScintillaWin.cxx
index 06ee43483..74ce8fb41 100644
--- a/scintilla/win32/ScintillaWin.cxx
+++ b/scintilla/win32/ScintillaWin.cxx
@@ -71,6 +71,7 @@ Used by VSCode, Atom etc.
 #ifdef SCI_LEXER
 #include "SciLexer.h"
 #endif
+#include "CharacterCategory.h"
 #ifdef SCI_LEXER
 #include "LexerModule.h"
 #endif
diff --git a/src/Edit.c b/src/Edit.c
index b64a75cc9..400291688 100644
--- a/src/Edit.c
+++ b/src/Edit.c
@@ -1293,6 +1293,8 @@ bool EditLoadFile(
   Encoding_SrcCmdLn(CPI_NONE);
   Encoding_SrcWeak(CPI_NONE);
 
+  SciCall_SetCharacterCategoryOptimization(Encoding_IsCJK(iAnalyzedEncoding) ? 0x10000 : 0x1000);
+
   return true;
 }
 
diff --git a/src/Encoding.c b/src/Encoding.c
index 9a7bb0364..d90a04335 100644
--- a/src/Encoding.c
+++ b/src/Encoding.c
@@ -507,6 +507,11 @@ bool Encoding_IsDefault(const cpi_enc_t iEncoding) {
 }
 // ============================================================================
 
+bool Encoding_IsASCII(const cpi_enc_t iEncoding) {
+  return  (iEncoding >= 0) ? (g_Encodings[iEncoding].uFlags & NCP_ASCII_7BIT) : false;
+}
+// ============================================================================
+
 bool Encoding_IsANSI(const cpi_enc_t iEncoding) {
   return  (iEncoding >= 0) ? (g_Encodings[iEncoding].uFlags & NCP_ANSI) : false;
 }
@@ -532,12 +537,36 @@ bool Encoding_IsMBCS(const cpi_enc_t iEncoding) {
 }
 // ============================================================================
 
-bool Encoding_IsASCII(const cpi_enc_t iEncoding) {
-  return  (iEncoding >= 0) ? (g_Encodings[iEncoding].uFlags & NCP_ASCII_7BIT) : false;
+bool Encoding_IsCJK(const cpi_enc_t iEncoding) {
+  UINT const codePage = Encoding_GetCodePage(iEncoding);
+  switch (codePage) 
+  {
+  case 932:
+  case 936:
+  case 949:
+  case 950:
+  case 951:
+  case 1361:
+  case 10001:
+  case 10002:
+  case 10003:
+  case 10008:
+  case 20000:
+  case 20932:
+  case 20936:
+  case 50220:
+  case 50225:
+  case 51949:
+  case 52936:
+  case 54936:
+    return true;
+  default:
+    break;
+  }
+  return false;
 }
 // ============================================================================
 
-
 bool Encoding_IsUNICODE(const cpi_enc_t iEncoding) {
   return  (iEncoding >= 0) ? (g_Encodings[iEncoding].uFlags & NCP_UNICODE) : false;
 }
diff --git a/src/Encoding.h b/src/Encoding.h
index 576e8e715..17d88f875 100644
--- a/src/Encoding.h
+++ b/src/Encoding.h
@@ -88,12 +88,13 @@ bool       Encoding_GetFromComboboxEx(HWND hwnd, cpi_enc_t* pidEncoding);
 UINT       Encoding_GetCodePage(const cpi_enc_t iEncoding);
            
 bool       Encoding_IsDefault(const cpi_enc_t iEncoding);
+bool       Encoding_IsASCII(const cpi_enc_t iEncoding);
 bool       Encoding_IsANSI(const cpi_enc_t iEncoding);
 bool       Encoding_IsOEM(const cpi_enc_t iEncoding);
 bool       Encoding_IsUTF8(const cpi_enc_t iEncoding);
 bool       Encoding_IsUTF8_SIGN(const cpi_enc_t iEncoding);
 bool       Encoding_IsMBCS(const cpi_enc_t iEncoding);
-bool       Encoding_IsASCII(const cpi_enc_t iEncoding);
+bool       Encoding_IsCJK(const cpi_enc_t iEncoding);
 bool       Encoding_IsUNICODE(const cpi_enc_t iEncoding);
 bool       Encoding_IsUNICODE_BOM(const cpi_enc_t iEncoding);
 bool       Encoding_IsUNICODE_REVERSE(const cpi_enc_t iEncoding);
diff --git a/src/SciCall.h b/src/SciCall.h
index 82599d741..83006b3b3 100644
--- a/src/SciCall.h
+++ b/src/SciCall.h
@@ -467,7 +467,7 @@ DeclareSciCallV1(SetUndoCollection, SETUNDOCOLLECTION, bool, bCollectUndo)
 DeclareSciCallV1(SetBufferedDraw, SETBUFFEREDDRAW, bool, value)
 DeclareSciCallV1(SetTechnology, SETTECHNOLOGY, int, technology)
 DeclareSciCallV1(SetBidirectional, SETBIDIRECTIONAL, int, direction)
-
+DeclareSciCallV1(SetCharacterCategoryOptimization, SETCHARACTERCATEGORYOPTIMIZATION, int, count)
 
 //=============================================================================
 //
diff --git a/src/Version.h b/src/Version.h
index b279c7a4d..7425e60c6 100644
--- a/src/Version.h
+++ b/src/Version.h
@@ -62,7 +62,7 @@
 #if defined(_MSC_VER)
     #if (_MSC_VER >= 1920)
         #if(_MSC_FULL_VER >= 192027508)
-            #define VER_CPL     MS Visual C++ 2019 16.0.0 Prev 4.3
+            #define VER_CPL     MS Visual C++ 2019 16.0.0 Prev 4.(3-4)
         #elif(_MSC_FULL_VER >= 192027404)
             #define VER_CPL     MS Visual C++ 2019 16.0.0 Prev 4.(0-2)
         #elif(_MSC_FULL_VER >= 192027323)