From a2c5bbfb20280698006470ceb2eda0c60e15708a Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Mon, 14 Dec 2015 18:36:20 +0900
Subject: [PATCH 01/18] Conform ampersand-error reporting to HTML spec

---
 .../validator/htmlparser/impl/Tokenizer.java  | 54 ++++++++++---------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index 318df252..6a90a8df 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -220,6 +220,8 @@ public class Tokenizer implements Locator {
 
     public static final int PROCESSING_INSTRUCTION_QUESTION_MARK = 74;
 
+    public static final int AMBIGUOUS_AMPERSAND = 75;
+
     /**
      * Magic value for UTF-16 operations.
      */
@@ -3106,6 +3108,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                         case '<':
                         case '&':
                         case '\u0000':
+                        case ';':
                             emitOrAppendCharRefBuf(returnState);
                             if ((returnState & DATA_AND_RCDATA_MASK) == 0) {
                                 cstart = pos;
@@ -3134,17 +3137,12 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 firstCharKey = c - 'A';
                             } else {
                                 // No match
-                                /*
-                                 * If no match can be made, then this is a parse
-                                 * error.
-                                 */
-                                errNoNamedCharacterMatch();
                                 emitOrAppendCharRefBuf(returnState);
                                 if ((returnState & DATA_AND_RCDATA_MASK) == 0) {
                                     cstart = pos;
                                 }
                                 reconsume = true;
-                                state = transition(state, returnState, reconsume, pos);
+                                state = transition(state, Tokenizer.AMBIGUOUS_AMPERSAND, reconsume, pos);
                                 continue stateloop;
                             }
                             // Didn't fail yet
@@ -3205,17 +3203,12 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                             }
                         }
                         if (hilo == 0) {
-                            /*
-                             * If no match can be made, then this is a parse
-                             * error.
-                             */
-                            errNoNamedCharacterMatch();
                             emitOrAppendCharRefBuf(returnState);
                             if ((returnState & DATA_AND_RCDATA_MASK) == 0) {
                                 cstart = pos;
                             }
                             reconsume = true;
-                            state = transition(state, returnState, reconsume, pos);
+                            state = transition(state, Tokenizer.AMBIGUOUS_AMPERSAND, reconsume, pos);
                             continue stateloop;
                         }
                         // Didn't fail yet
@@ -3298,16 +3291,12 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
 
                     if (candidate == -1) {
                         // reconsume deals with CR, LF or nul
-                        /*
-                         * If no match can be made, then this is a parse error.
-                         */
-                        errNoNamedCharacterMatch();
                         emitOrAppendCharRefBuf(returnState);
                         if ((returnState & DATA_AND_RCDATA_MASK) == 0) {
                             cstart = pos;
                         }
                         reconsume = true;
-                        state = transition(state, returnState, reconsume, pos);
+                        state = transition(state, Tokenizer.AMBIGUOUS_AMPERSAND, reconsume, pos);
                         continue stateloop;
                     } else {
                         // c can't be CR, LF or nul if we got here
@@ -3345,10 +3334,9 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                      * after the U+0026 AMPERSAND (&) must be
                                      * unconsumed, and nothing is returned.
                                      */
-                                    errNoNamedCharacterMatch();
                                     appendCharRefBufToStrBuf();
                                     reconsume = true;
-                                    state = transition(state, returnState, reconsume, pos);
+                                    state = transition(state, Tokenizer.AMBIGUOUS_AMPERSAND, reconsume, pos);
                                     continue stateloop;
                                 }
                             }
@@ -3411,6 +3399,28 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                          * I'm ∉ I tell you.
                          */
                     }
+                    // XXX reorder point
+                case AMBIGUOUS_AMPERSAND:
+                    ampersandloop: for (;;) {
+                        if (reconsume) {
+                            if (++pos == endPos) {
+                                break stateloop;
+                            }
+                            pos--;
+                            c = checkChar(buf, pos);
+                        }
+                        if (c == ';') {
+                            errNoNamedCharacterMatch();
+                        } else if ((c >= '0' && c <= '9')
+                                || (c >= 'A' && c <= 'Z')
+                                || (c >= 'a' && c <= 'z')) {
+                            appendStrBuf(c);
+                            pos++;
+                            continue;
+                        }
+                        state = transition(state, returnState, reconsume, pos);
+                        continue stateloop;
+                    }
                 case CONSUME_NCR:
                     if (++pos == endPos) {
                         break stateloop;
@@ -6501,7 +6511,6 @@ public void eof() throws SAXException {
                     state = returnState;
                     continue;
                 case CHARACTER_REFERENCE_HILO_LOOKUP:
-                    errNoNamedCharacterMatch();
                     emitOrAppendCharRefBuf(returnState);
                     state = returnState;
                     continue;
@@ -6555,10 +6564,6 @@ public void eof() throws SAXException {
                     }
 
                     if (candidate == -1) {
-                        /*
-                         * If no match can be made, then this is a parse error.
-                         */
-                        errNoNamedCharacterMatch();
                         emitOrAppendCharRefBuf(returnState);
                         state = returnState;
                         continue eofloop;
@@ -6596,7 +6601,6 @@ public void eof() throws SAXException {
                                      * after the U+0026 AMPERSAND (&) must be
                                      * unconsumed, and nothing is returned.
                                      */
-                                    errNoNamedCharacterMatch();
                                     appendCharRefBufToStrBuf();
                                     state = returnState;
                                     continue eofloop;

From 2b06e4df9d7480339918468758734cfceb425a60 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Mon, 10 Nov 2014 16:15:26 +0900
Subject: [PATCH 02/18] Fix grammar problem in HTML parser error message

---
 src/nu/validator/htmlparser/impl/TreeBuilder.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index a81f144a..b6be15ff 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -6621,7 +6621,7 @@ private void errFooSeenWhenFooOpen(@Local String name) throws SAXException {
         if (errorHandler == null) {
             return;
         }
-        errNoCheck("An \u201C" + name + "\u201D start tag seen but an element of the same type was already open.");
+        errNoCheck("Start tag \u201C" + name + "\u201D seen but an element of the same type was already open.");
     }
 
     private void errHeadingWhenHeadingOpen() throws SAXException {

From 8bd0b78468416171a5772118716e8d852c1b8712 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Mon, 14 Dec 2015 19:06:01 +0900
Subject: [PATCH 03/18] Fix "non-space characters insided a table" typo

---
 src/nu/validator/htmlparser/impl/TreeBuilder.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index b6be15ff..de3edfeb 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -6712,7 +6712,7 @@ private void errNoTableRowToClose() throws SAXException {
     }
 
     private void errNonSpaceInTable() throws SAXException {
-        err("Misplaced non-space characters insided a table.");
+        err("Misplaced non-space characters inside a table.");
     }
 
     private void errUnclosedChildrenInRuby() throws SAXException {

From 8fac68b4f429c99d6a1f05ede3819b6fd8bf44ee Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Sat, 30 Jan 2016 20:19:10 +0900
Subject: [PATCH 04/18] Report 1024 as byte limit for meta charset sniff

Fixes https://github.com/validator/validator/issues/232
---
 src/nu/validator/htmlparser/impl/Tokenizer.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index 6a90a8df..95068875 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -1289,7 +1289,7 @@ private void addAttributeWithoutValue() throws SAXException {
         // [NOCPP[
         if (metaBoundaryPassed && AttributeName.CHARSET == attributeName
                 && ElementName.META == tagName) {
-            err("A \u201Ccharset\u201D attribute on a \u201Cmeta\u201D element found after the first 512 bytes.");
+            err("A \u201Ccharset\u201D attribute on a \u201Cmeta\u201D element found after the first 1024 bytes.");
         }
         // ]NOCPP]
         if (attributeName != null) {
@@ -1337,7 +1337,7 @@ private void addAttributeWithValue() throws SAXException {
         // [NOCPP[
         if (metaBoundaryPassed && ElementName.META == tagName
                 && AttributeName.CHARSET == attributeName) {
-            err("A \u201Ccharset\u201D attribute on a \u201Cmeta\u201D element found after the first 512 bytes.");
+            err("A \u201Ccharset\u201D attribute on a \u201Cmeta\u201D element found after the first 1024 bytes.");
         }
         // ]NOCPP]
         if (attributeName != null) {

From 20eaf92512f66ae83f678e3c0bee1f0d407cfab0 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Fri, 10 Jun 2016 21:39:28 +0900
Subject: [PATCH 05/18] Make consecutive hyphens in comments a non-error

Also allow `<!-->` at (IE conditional) comment end

See https://github.com/whatwg/html/pull/1356
See https://github.com/whatwg/html/pull/1456
---
 .../impl/ErrorReportingTokenizer.java         |  10 +-
 .../validator/htmlparser/impl/Tokenizer.java  | 199 ++++++++++++++++--
 2 files changed, 190 insertions(+), 19 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java b/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java
index 9db9e8a7..7494fb28 100644
--- a/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java
+++ b/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2009-2013 Mozilla Foundation
+ * Copyright (c) 2009-2017 Mozilla Foundation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -413,8 +413,8 @@ private boolean isAstralPrivateUse(int c) {
         err("Nameless doctype.");
     }
 
-    @Override protected void errConsecutiveHyphens() throws SAXException {
-        err("Consecutive hyphens did not terminate a comment. \u201C--\u201D is not permitted inside a comment, but e.g. \u201C- -\u201D is.");
+    @Override protected void errNestedComment() throws SAXException {
+        err("Saw \u201C<!--\u201D within a comment. Probable cause: Nested comment (not allowed).");
     }
 
     @Override protected void errPrematureEndOfComment() throws SAXException {
@@ -712,10 +712,6 @@ private boolean isAstralPrivateUse(int c) {
         err("Missing space before doctype name.");
     }
 
-    @Override protected void errHyphenHyphenBang() throws SAXException {
-        err("\u201C--!\u201D found in comment.");
-    }
-
     @Override protected void errNcrControlChar() throws SAXException {
         err("Character reference expands to a control character ("
                 + toUPlusString((char) value) + ").");
diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index 95068875..c8e1828a 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -222,6 +222,14 @@ public class Tokenizer implements Locator {
 
     public static final int AMBIGUOUS_AMPERSAND = 75;
 
+    public static final int COMMENT_LESSTHAN = 76;
+
+    public static final int COMMENT_LESSTHAN_BANG = 77;
+
+    public static final int COMMENT_LESSTHAN_BANG_DASH = 78;
+
+    public static final int COMMENT_LESSTHAN_BANG_DASH_DASH = 79;
+
     /**
      * Magic value for UTF-16 operations.
      */
@@ -1034,9 +1042,8 @@ private void maybeAppendSpaceToBogusComment() throws SAXException {
 
     // ]NOCPP]
 
-    @Inline private void adjustDoubleHyphenAndAppendToStrBufAndErr(char c)
+    @Inline private void adjustDoubleHyphenAndAppendToStrBufAndErr(char c, boolean reportedConsecutiveHyphens)
             throws SAXException {
-        errConsecutiveHyphens();
         // [NOCPP[
         switch (commentPolicy) {
             case ALTER_INFOSET:
@@ -1047,7 +1054,9 @@ private void maybeAppendSpaceToBogusComment() throws SAXException {
                 appendStrBuf('-');
                 // CPPONLY: MOZ_FALLTHROUGH;
             case ALLOW:
-                warn("The document is not mappable to XML 1.0 due to two consecutive hyphens in a comment.");
+                if (!reportedConsecutiveHyphens) {
+                    warn("The document is not mappable to XML 1.0 due to two consecutive hyphens in a comment.");
+                }
                 // ]NOCPP]
                 appendStrBuf(c);
                 // [NOCPP[
@@ -1509,6 +1518,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
     @SuppressWarnings("unused") private int stateLoop(int state, char c,
             int pos, @NoLength char[] buf, boolean reconsume, int returnState,
             int endPos) throws SAXException {
+        boolean reportedConsecutiveHyphens = false;
         /*
          * Idioms used in this code:
          *
@@ -2594,6 +2604,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                     }
                     // CPPONLY: MOZ_FALLTHROUGH;
                 case COMMENT_START:
+                    reportedConsecutiveHyphens = false;
                     commentstartloop: for (;;) {
                         if (++pos == endPos) {
                             break stateloop;
@@ -2626,6 +2637,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                  */
                                 state = transition(state, Tokenizer.DATA, reconsume, pos);
                                 continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
                             case '\r':
                                 appendStrBufCarriageReturn();
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
@@ -2671,6 +2686,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 state = transition(state, Tokenizer.COMMENT_END_DASH, reconsume, pos);
                                 break commentloop;
                             // continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
                             case '\r':
                                 appendStrBufCarriageReturn();
                                 break stateloop;
@@ -2713,6 +2732,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 state = transition(state, Tokenizer.COMMENT_END, reconsume, pos);
                                 break commentenddashloop;
                             // continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
                             case '\r':
                                 appendStrBufCarriageReturn();
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
@@ -2767,11 +2790,16 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                  * Append a U+002D HYPHEN-MINUS (-) character to
                                  * the comment token's data.
                                  */
-                                adjustDoubleHyphenAndAppendToStrBufAndErr(c);
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
                                 /*
                                  * Stay in the comment end state.
                                  */
                                 continue;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
                             case '\r':
                                 adjustDoubleHyphenAndAppendToStrBufCarriageReturn();
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
@@ -2781,7 +2809,6 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
                                 continue stateloop;
                             case '!':
-                                errHyphenHyphenBang();
                                 appendStrBuf(c);
                                 state = transition(state, Tokenizer.COMMENT_END_BANG, reconsume, pos);
                                 continue stateloop;
@@ -2794,7 +2821,8 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                  * and the input character to the comment
                                  * token's data.
                                  */
-                                adjustDoubleHyphenAndAppendToStrBufAndErr(c);
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
                                 /*
                                  * Switch to the comment state.
                                  */
@@ -2864,6 +2892,148 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue stateloop;
                         }
                     }
+                case COMMENT_LESSTHAN:
+                    for (;;) {
+                        if (++pos == endPos) {
+                            break stateloop;
+                        }
+                        c = checkChar(buf, pos);
+                        switch (c) {
+                            case '!':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN_BANG, reconsume, pos);
+                                continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
+                            case '-':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_END_DASH, reconsume, pos);
+                                continue stateloop;
+                            case '\r':
+                                appendStrBufCarriageReturn();
+                                break stateloop;
+                            case '\n':
+                                appendStrBufLineFeed();
+                                continue;
+                            case '\u0000':
+                                c = '\uFFFD';
+                                // fall thru
+                            default:
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue stateloop;
+                        }
+                    }
+                case COMMENT_LESSTHAN_BANG:
+                    for (;;) {
+                        if (++pos == endPos) {
+                            break stateloop;
+                        }
+                        c = checkChar(buf, pos);
+                        switch (c) {
+                            case '-':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN_BANG_DASH, reconsume, pos);
+                                continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
+                            case '\r':
+                                appendStrBufCarriageReturn();
+                                break stateloop;
+                            case '\n':
+                                appendStrBufLineFeed();
+                                continue;
+                            case '\u0000':
+                                c = '\uFFFD';
+                                // fall thru
+                            default:
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue stateloop;
+                        }
+                    }
+                case COMMENT_LESSTHAN_BANG_DASH:
+                    for (;;) {
+                        if (++pos == endPos) {
+                            break stateloop;
+                        }
+                        c = checkChar(buf, pos);
+                        switch (c) {
+                            case '-':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN_BANG_DASH_DASH, reconsume, pos);
+                                continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
+                            case '\r':
+                                appendStrBufCarriageReturn();
+                                break stateloop;
+                            case '\n':
+                                appendStrBufLineFeed();
+                                continue;
+                            case '\u0000':
+                                c = '\uFFFD';
+                                // fall thru
+                            default:
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue stateloop;
+                        }
+                    }
+                case COMMENT_LESSTHAN_BANG_DASH_DASH:
+                    for (;;) {
+                        if (++pos == endPos) {
+                            break stateloop;
+                        }
+                        c = checkChar(buf, pos);
+                        switch (c) {
+                            case '>':
+                                appendStrBuf(c);
+                                emitComment(3, pos);
+                                state = transition(state, Tokenizer.DATA, reconsume, pos);
+                                continue stateloop;
+                            case '-':
+                                errNestedComment();
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
+                                state = transition(state, Tokenizer.COMMENT_END, reconsume, pos);
+                                continue stateloop;
+                            case '\r':
+                                errNestedComment();
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                break stateloop;
+                            case '\n':
+                                errNestedComment();
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue;
+                            case '\u0000':
+                                c = '\uFFFD';
+                                // fall thru
+                            case '!':
+                                errNestedComment();
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
+                                state = transition(state, Tokenizer.COMMENT_END_BANG, reconsume, pos);
+                                continue stateloop;
+                            default:
+                                errNestedComment();
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
+                                reportedConsecutiveHyphens = true;
+                                state = transition(state, Tokenizer.COMMENT_END, reconsume, pos);
+                                continue stateloop;
+                        }
+                    }
+                    // XXX reorder point
                 case COMMENT_START_DASH:
                     if (++pos == endPos) {
                         break stateloop;
@@ -2892,6 +3062,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                              */
                             state = transition(state, Tokenizer.DATA, reconsume, pos);
                             continue stateloop;
+                        case '<':
+                            appendStrBuf(c);
+                            state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                            continue stateloop;
                         case '\r':
                             appendStrBufCarriageReturn();
                             state = transition(state, Tokenizer.COMMENT, reconsume, pos);
@@ -6019,13 +6193,13 @@ private void initDoctypeFields() {
     @Inline private void adjustDoubleHyphenAndAppendToStrBufCarriageReturn()
             throws SAXException {
         silentCarriageReturn();
-        adjustDoubleHyphenAndAppendToStrBufAndErr('\n');
+        adjustDoubleHyphenAndAppendToStrBufAndErr('\n', false);
     }
 
     @Inline private void adjustDoubleHyphenAndAppendToStrBufLineFeed()
             throws SAXException {
         silentLineFeed();
-        adjustDoubleHyphenAndAppendToStrBufAndErr('\n');
+        adjustDoubleHyphenAndAppendToStrBufAndErr('\n', false);
     }
 
     @Inline private void appendStrBufLineFeed() {
@@ -6330,6 +6504,8 @@ public void eof() throws SAXException {
                     break eofloop;
                 case COMMENT_START:
                 case COMMENT:
+                case COMMENT_LESSTHAN:
+                case COMMENT_LESSTHAN_BANG:
                     /*
                      * EOF Parse error.
                      */
@@ -6341,6 +6517,7 @@ public void eof() throws SAXException {
                      */
                     break eofloop;
                 case COMMENT_END:
+                case COMMENT_LESSTHAN_BANG_DASH_DASH:
                     errEofInComment();
                     /* Emit the comment token. */
                     emitComment(2, 0);
@@ -6350,6 +6527,7 @@ public void eof() throws SAXException {
                     break eofloop;
                 case COMMENT_END_DASH:
                 case COMMENT_START_DASH:
+                case COMMENT_LESSTHAN_BANG_DASH:
                     errEofInComment();
                     /* Emit the comment token. */
                     emitComment(1, 0);
@@ -6974,7 +7152,7 @@ protected void errGtInPublicId() throws SAXException {
     protected void errNamelessDoctype() throws SAXException {
     }
 
-    protected void errConsecutiveHyphens() throws SAXException {
+    protected void errNestedComment() throws SAXException {
     }
 
     protected void errPrematureEndOfComment() throws SAXException {
@@ -7124,9 +7302,6 @@ protected void errExpectedSystemId() throws SAXException {
     protected void errMissingSpaceBeforeDoctypeName() throws SAXException {
     }
 
-    protected void errHyphenHyphenBang() throws SAXException {
-    }
-
     protected void errNcrControlChar() throws SAXException {
     }
 

From 82d4913a662e61c9e6cab2618311bc1070c4a2eb Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Wed, 20 Jul 2016 00:15:36 +0900
Subject: [PATCH 06/18] =?UTF-8?q?Correct=20error=20for=20EOF=20in=20?=
 =?UTF-8?q?=E2=80=9Cin=20template=E2=80=9D=20state?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Doing `errUnclosedElements(eltPos, "template")` for EOF in the “in
template” state results in the error message “End tag `template` seen, but
there were open elements”, which is all wrong because the actual problem is
that though a `template` end tag was expected, EOF was reached without a
`template` end tag being seen.

So let’s instead when we reach this just report the list of open elements.
---
 src/nu/validator/htmlparser/impl/TreeBuilder.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index de3edfeb..790a9c28 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -1567,7 +1567,7 @@ public final void eof() throws SAXException {
                         break eofloop;
                     }
                     if (errorHandler != null) {
-                        errUnclosedElements(eltPos, "template");
+                        errListUnclosedStartTags(0);
                     }
                     while (currentPtr >= eltPos) {
                         pop();

From 4a37fe9aebc93b23f2f5290ed64e6b10cbba7cd6 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Wed, 16 Nov 2016 11:01:48 +0900
Subject: [PATCH 07/18] Report error always for Transitional doctype

Fixes https://github.com/validator/validator/issues/408 Thanks @still-dreaming-1
---
 src/nu/validator/htmlparser/impl/TreeBuilder.java | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index 790a9c28..373d7238 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -896,13 +896,7 @@ public final void doctype(@Local String name, String publicIdentifier,
                             warn("Comments seen before doctype. Internet Explorer will go into the quirks mode.",
                                     firstCommentLocation);
                         }
-                        if ("-//W3C//DTD HTML 4.01 Transitional//EN".equals(publicIdentifier)) {
-                            if (!"http://www.w3.org/TR/html4/loose.dtd".equals(systemIdentifier)) {
-                                warn("The doctype did not contain the system identifier prescribed by the HTML 4.01 specification. Expected \u201C<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">\u201D.");
-                            }
-                        } else {
-                            err("Almost standards mode doctype. Expected e.g. \u201C<!DOCTYPE html>\u201D.");
-                        }
+                        errAlmostStandardsDoctype();
                         documentModeInternal(
                                 DocumentMode.ALMOST_STANDARDS_MODE,
                                 publicIdentifier, systemIdentifier, html4);

From 303dcc8054ca1b5f694651b5a5c9d6ce7de40639 Mon Sep 17 00:00:00 2001
From: Simon Pieters <zcorpan@gmail.com>
Date: Tue, 30 Aug 2016 12:04:17 +0200
Subject: [PATCH 08/18] Remove warning about comments before doctype

This only applied to IE6..9, and does not apply to conditional comments.
---
 .../htmlparser/impl/TreeBuilder.java          | 40 -------------------
 1 file changed, 40 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index 373d7238..dbdc653b 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -391,8 +391,6 @@ public abstract class TreeBuilder<T> implements TokenHandler,
 
     private DoctypeExpectation doctypeExpectation = DoctypeExpectation.HTML;
 
-    private LocatorImpl firstCommentLocation;
-
     // ]NOCPP]
 
     private boolean scriptingEnabled = false;
@@ -614,7 +612,6 @@ final void warn(String message, Locator locator) throws SAXException {
         html4 = false;
         idLocations.clear();
         wantingComments = wantsComments();
-        firstCommentLocation = null;
         // ]NOCPP]
         start(fragment);
         charBufferLen = 0;
@@ -763,22 +760,12 @@ public final void doctype(@Local String name, String publicIdentifier,
                                 publicIdentifier, systemIdentifier, false);
                     } else if (isAlmostStandards(publicIdentifier,
                             systemIdentifier)) {
-                        // [NOCPP[
-                        if (firstCommentLocation != null) {
-                            warn("Comments seen before doctype. Internet Explorer will go into the quirks mode.",
-                                    firstCommentLocation);
-                        }
-                        // ]NOCPP]
                         errAlmostStandardsDoctype();
                         documentModeInternal(
                                 DocumentMode.ALMOST_STANDARDS_MODE,
                                 publicIdentifier, systemIdentifier, false);
                     } else {
                         // [NOCPP[
-                        if (firstCommentLocation != null) {
-                            warn("Comments seen before doctype. Internet Explorer will go into the quirks mode.",
-                                    firstCommentLocation);
-                        }
                         if ((Portability.literalEqualsString(
                                 "-//W3C//DTD HTML 4.0//EN", publicIdentifier) && (systemIdentifier == null || Portability.literalEqualsString(
                                 "http://www.w3.org/TR/REC-html40/strict.dtd",
@@ -821,19 +808,11 @@ public final void doctype(@Local String name, String publicIdentifier,
                                 publicIdentifier, systemIdentifier, true);
                     } else if (isAlmostStandards(publicIdentifier,
                             systemIdentifier)) {
-                        if (firstCommentLocation != null) {
-                            warn("Comments seen before doctype. Internet Explorer will go into the quirks mode.",
-                                    firstCommentLocation);
-                        }
                         err("Almost standards mode doctype. Expected \u201C<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/html4/strict.dtd\">\u201D.");
                         documentModeInternal(
                                 DocumentMode.ALMOST_STANDARDS_MODE,
                                 publicIdentifier, systemIdentifier, true);
                     } else {
-                        if (firstCommentLocation != null) {
-                            warn("Comments seen before doctype. Internet Explorer will go into the quirks mode.",
-                                    firstCommentLocation);
-                        }
                         if ("-//W3C//DTD HTML 4.01//EN".equals(publicIdentifier)) {
                             if (!"http://www.w3.org/TR/html4/strict.dtd".equals(systemIdentifier)) {
                                 warn("The doctype did not contain the system identifier prescribed by the HTML 4.01 specification. Expected \u201C<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/html4/strict.dtd\">\u201D.");
@@ -855,10 +834,6 @@ public final void doctype(@Local String name, String publicIdentifier,
                                 publicIdentifier, systemIdentifier, true);
                     } else if (isAlmostStandards(publicIdentifier,
                             systemIdentifier)) {
-                        if (firstCommentLocation != null) {
-                            warn("Comments seen before doctype. Internet Explorer will go into the quirks mode.",
-                                    firstCommentLocation);
-                        }
                         if ("-//W3C//DTD HTML 4.01 Transitional//EN".equals(publicIdentifier)
                                 && systemIdentifier != null) {
                             if (!"http://www.w3.org/TR/html4/loose.dtd".equals(systemIdentifier)) {
@@ -871,10 +846,6 @@ public final void doctype(@Local String name, String publicIdentifier,
                                 DocumentMode.ALMOST_STANDARDS_MODE,
                                 publicIdentifier, systemIdentifier, true);
                     } else {
-                        if (firstCommentLocation != null) {
-                            warn("Comments seen before doctype. Internet Explorer will go into the quirks mode.",
-                                    firstCommentLocation);
-                        }
                         err("The doctype was not the HTML 4.01 Transitional doctype. Expected \u201C<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">\u201D.");
                         documentModeInternal(DocumentMode.STANDARDS_MODE,
                                 publicIdentifier, systemIdentifier, true);
@@ -892,19 +863,11 @@ public final void doctype(@Local String name, String publicIdentifier,
                                 publicIdentifier, systemIdentifier, html4);
                     } else if (isAlmostStandards(publicIdentifier,
                             systemIdentifier)) {
-                        if (firstCommentLocation != null) {
-                            warn("Comments seen before doctype. Internet Explorer will go into the quirks mode.",
-                                    firstCommentLocation);
-                        }
                         errAlmostStandardsDoctype();
                         documentModeInternal(
                                 DocumentMode.ALMOST_STANDARDS_MODE,
                                 publicIdentifier, systemIdentifier, html4);
                     } else {
-                        if (firstCommentLocation != null) {
-                            warn("Comments seen before doctype. Internet Explorer will go into the quirks mode.",
-                                    firstCommentLocation);
-                        }
                         if ("-//W3C//DTD HTML 4.01//EN".equals(publicIdentifier)) {
                             if (!"http://www.w3.org/TR/html4/strict.dtd".equals(systemIdentifier)) {
                                 warn("The doctype did not contain the system identifier prescribed by the HTML 4.01 specification. Expected \u201C<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/html4/strict.dtd\">\u201D.");
@@ -978,9 +941,6 @@ public final void comment(@NoLength char[] buf, int start, int length)
             throws SAXException {
         needToDropLF = false;
         // [NOCPP[
-        if (firstCommentLocation == null) {
-            firstCommentLocation = new LocatorImpl(tokenizer);
-        }
         if (!wantingComments) {
             return;
         }

From 751b6786e39019ea62651def1a6ba81cf398c1bc Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Fri, 19 May 2017 00:32:05 +0900
Subject: [PATCH 09/18] =?UTF-8?q?Support=20=E2=80=9Cgenerate=20all=20impli?=
 =?UTF-8?q?ed=20end=20tags=20thoroughly=E2=80=9D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When the parser encounters a `</template>` end tag and there are other
open elements, the HTML spec requires the parser to “generate all
implied end tags thoroughly”, which unlike “generate implied end tags”
also includes generating implied end tags for table-parts elements
(caption, colgroup, tbody, thead, tfoot, td, th, and tr).
---
 .../htmlparser/impl/TreeBuilder.java          | 25 ++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index dbdc653b..ade6edaa 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -4089,7 +4089,7 @@ private void endTagTemplateInHead() throws SAXException {
             errStrayEndTag("template");
             return;
         }
-        generateImpliedEndTags();
+        generateImpliedEndTagsThoroughly();
         if (errorHandler != null && !isCurrent("template")) {
             errUnclosedElements(eltPos, "template");
         }
@@ -4230,6 +4230,29 @@ private void generateImpliedEndTags() throws SAXException {
         }
     }
 
+    private void generateImpliedEndTagsThoroughly() throws SAXException {
+        for (;;) {
+            switch (stack[currentPtr].getGroup()) {
+                case CAPTION:
+                case COLGROUP:
+                case DD_OR_DT:
+                case LI:
+                case OPTGROUP:
+                case OPTION:
+                case P:
+                case RB_OR_RTC:
+                case RT_OR_RP:
+                case TBODY_OR_THEAD_OR_TFOOT:
+                case TD_OR_TH:
+                case TR:
+                    pop();
+                    continue;
+                default:
+                    return;
+            }
+        }
+    }
+
     private boolean isSecondOnStackBody() {
         return currentPtr >= 1 && stack[1].getGroup() == TreeBuilder.BODY;
     }

From 77503d994da0d1473bc1b3dfcd8e05e2ad27a46a Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Thu, 18 Jan 2018 13:19:36 +0900
Subject: [PATCH 10/18] Drop parse error for missing </caption> end tag

See https://html.spec.whatwg.org/multipage/parsing.html#parsing-main-incaption:stack-of-open-elements-2
The parse error for this case was dropped from the spec in
https://github.com/whatwg/html/commit/759ee62

Relates to https://github.com/validator/validator/issues/593
and https://bugzilla.mozilla.org/show_bug.cgi?id=1429415
---
 src/nu/validator/htmlparser/impl/TreeBuilder.java | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index ade6edaa..87da81e0 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -1970,9 +1970,10 @@ public final void startTag(ElementName elementName,
                         case TBODY_OR_THEAD_OR_TFOOT:
                         case TR:
                         case TD_OR_TH:
-                            errStrayStartTag(name);
                             eltPos = findLastInTableScope("caption");
                             if (eltPos == TreeBuilder.NOT_FOUND_ON_STACK) {
+                                assert fragment || isTemplateContents();
+                                errStrayStartTag(name);
                                 break starttagloop;
                             }
                             generateImpliedEndTags();
@@ -3464,9 +3465,11 @@ public final void endTag(ElementName elementName) throws SAXException {
                             mode = IN_TABLE;
                             break endtagloop;
                         case TABLE:
-                            errTableClosedWhileCaptionOpen();
                             eltPos = findLastInTableScope("caption");
+
                             if (eltPos == TreeBuilder.NOT_FOUND_ON_STACK) {
+                                assert fragment || isTemplateContents();
+                                errStrayEndTag(name);
                                 break endtagloop;
                             }
                             generateImpliedEndTags();
@@ -6680,10 +6683,6 @@ private void errHtmlStartTagInForeignContext(@Local String name)
                 + "\u201D in a foreign namespace context.");
     }
 
-    private void errTableClosedWhileCaptionOpen() throws SAXException {
-        err("\u201Ctable\u201D closed but \u201Ccaption\u201D was still open.");
-    }
-
     private void errNoTableRowToClose() throws SAXException {
         err("No table row to close.");
     }

From 4a68e5c1717bd946a2269d78dc0757fa19feeca1 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Mon, 19 Feb 2018 12:16:55 +0900
Subject: [PATCH 11/18] Require UTF-8

See https://html.spec.whatwg.org/multipage/semantics.html#charset:utf-8
and https://github.com/whatwg/html/commit/fae77e3
---
 src/nu/validator/htmlparser/io/Driver.java                | 8 ++++----
 src/nu/validator/htmlparser/io/HtmlInputStreamReader.java | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/nu/validator/htmlparser/io/Driver.java b/src/nu/validator/htmlparser/io/Driver.java
index f0b0cc55..a3277ea8 100644
--- a/src/nu/validator/htmlparser/io/Driver.java
+++ b/src/nu/validator/htmlparser/io/Driver.java
@@ -197,9 +197,9 @@ public void tokenize(InputSource is) throws SAXException, IOException {
                         tokenizer.getErrorHandler(), tokenizer, this, heuristics);
             } else {
                 if (this.characterEncoding != Encoding.UTF8) {
-                    warnWithoutLocation("Legacy encoding \u201C"
+                    errorWithoutLocation("Legacy encoding \u201C"
                             + this.characterEncoding.getCanonName()
-                            + "\u201D used. Documents should use UTF-8.");
+                            + "\u201D used. Documents must use UTF-8.");
                 }
                 becomeConfident();
                 this.reader = new HtmlInputStreamReader(inputStream,
@@ -416,14 +416,14 @@ public void setHeuristics(Heuristics heuristics) {
      *            the message
      * @throws SAXException
      */
-    protected void warnWithoutLocation(String message) throws SAXException {
+    protected void errorWithoutLocation(String message) throws SAXException {
         ErrorHandler errorHandler = tokenizer.getErrorHandler();
         if (errorHandler == null) {
             return;
         }
         SAXParseException spe = new SAXParseException(message, null,
                 tokenizer.getSystemId(), -1, -1);
-        errorHandler.warning(spe);
+        errorHandler.error(spe);
     }
 
     /**
diff --git a/src/nu/validator/htmlparser/io/HtmlInputStreamReader.java b/src/nu/validator/htmlparser/io/HtmlInputStreamReader.java
index 413f0d9e..63884326 100755
--- a/src/nu/validator/htmlparser/io/HtmlInputStreamReader.java
+++ b/src/nu/validator/htmlparser/io/HtmlInputStreamReader.java
@@ -131,9 +131,9 @@ public HtmlInputStreamReader(InputStream inputStream,
             if (encoding == null) {
                 declared = false;
             } else if (encoding != Encoding.UTF8) {
-                warn("Legacy encoding \u201C"
+                err("Legacy encoding \u201C"
                         + encoding.getCanonName()
-                        + "\u201D used. Documents should use UTF-8.");
+                        + "\u201D used. Documents must use UTF-8.");
             }
             if (encoding == null
                     && (heuristics == Heuristics.CHARDET || heuristics == Heuristics.ALL)) {
@@ -160,9 +160,9 @@ public HtmlInputStreamReader(InputStream inputStream,
                     driver.setEncoding(Encoding.UTF8, Confidence.CERTAIN);
                 }
             } else {
-                warn("Legacy encoding \u201C"
+                err("Legacy encoding \u201C"
                         + encoding.getCanonName()
-                        + "\u201D used. Documents should use UTF-8.");
+                        + "\u201D used. Documents must use UTF-8.");
                 if (driver != null) {
                     driver.setEncoding(Encoding.UTF16, Confidence.CERTAIN);
                 }

From 84337162bd85b0d1146f27bba4394fcec7480634 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Fri, 16 Mar 2018 12:30:10 +0900
Subject: [PATCH 12/18] Add suppress "unchecked" for a TreeBuilder method

In `nu.validator.htmlparser.impl.TreeBuilder`, this change adds
`@SuppressWarnings("unchecked")` to the `getUnusedStackNode()` method.
---
 src/nu/validator/htmlparser/impl/TreeBuilder.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index 87da81e0..ccdc3b92 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -5020,7 +5020,7 @@ void notifyUnusedStackNode(int idxInStackNodes) {
         }
     }
 
-    private StackNode<T> getUnusedStackNode() {
+    @SuppressWarnings("unchecked") private StackNode<T> getUnusedStackNode() {
         // Search for an unused stack node.
         while (stackNodesIdx < numStackNodes) {
             if (stackNodes[stackNodesIdx].isUnused()) {

From f475aa6653a279e1d3ae81353bcbf180936ae3a4 Mon Sep 17 00:00:00 2001
From: Matt Brundage <1263178+mattbrundage@users.noreply.github.com>
Date: Thu, 7 Feb 2019 10:28:55 -0500
Subject: [PATCH 13/18] Fix typo in error message about over-deep tree

---
 src/nu/validator/htmlparser/impl/TreeBuilder.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index ccdc3b92..568903c3 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -6450,7 +6450,7 @@ public int getTemplateModeStackLength() {
      * @throws SAXException
      */
     private void errDeepTree() throws SAXException {
-        err("The document tree is more than 513 elements deep, which causes Firefox and Chrome flatten the tree.");
+        err("The document tree is more than 513 elements deep, which causes Firefox and Chrome to flatten the tree.");
     }
 
     /**

From 291b51e709880cd6363f5f0e8d662ac0a6a722f1 Mon Sep 17 00:00:00 2001
From: Simon Pieters <zcorpan@gmail.com>
Date: Wed, 6 Mar 2019 16:20:08 +0100
Subject: [PATCH 14/18] Improve message: bad start tag in noscript in head

---
 src/nu/validator/htmlparser/impl/TreeBuilder.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index 568903c3..860f6f5b 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -2585,7 +2585,7 @@ public final void startTag(ElementName elementName,
                             errFooSeenWhenFooOpen(name);
                             break starttagloop;
                         default:
-                            errBadStartTagInHead(name);
+                            errBadStartTagInNoscriptInHead(name);
                             pop();
                             mode = IN_HEAD;
                             continue;
@@ -6585,12 +6585,12 @@ private void errStartTagWithSelectOpen(@Local String name)
                 + "\u201D start tag with \u201Cselect\u201D open.");
     }
 
-    private void errBadStartTagInHead(@Local String name) throws SAXException {
+    private void errBadStartTagInNoscriptInHead(@Local String name) throws SAXException {
         if (errorHandler == null) {
             return;
         }
         errNoCheck("Bad start tag in \u201C" + name
-                + "\u201D in \u201Chead\u201D.");
+                + "\u201D in \u201Cnoscript\u201D in \u201Chead\u201D.");
     }
 
     private void errImage() throws SAXException {

From c4935b1e4a0d3162802bcb5cbb966d09db1fbf53 Mon Sep 17 00:00:00 2001
From: Mike Bremford <mike@bfo.co.uk>
Date: Wed, 15 Jan 2020 02:41:35 +0000
Subject: [PATCH 15/18] Ensure every Locator is also a Locator2 (#10)

* Replace org.xml.sax.helpers.LocatorImpl with nu.validator.htmlparser.impl.LocatorImpl

* instanceof before cast to Locator2
---
 .../htmlparser/impl/LocatorImpl.java          | 26 +++++++++++++++-
 .../validator/htmlparser/impl/Tokenizer.java  | 21 ++++++++++++-
 src/nu/validator/htmlparser/io/Driver.java    |  2 +-
 .../htmlparser/io/HtmlInputStreamReader.java  | 17 +++++++++-
 .../validator/htmlparser/io/MetaSniffer.java  | 17 +++++++++-
 .../validator/saxtree/DocumentFragment.java   |  2 +-
 src/nu/validator/saxtree/LocatorImpl.java     | 31 ++++++++++++++++++-
 src/nu/validator/saxtree/Node.java            | 28 ++++++++++++++++-
 src/nu/validator/saxtree/TreeParser.java      | 26 ++++++++++++++--
 9 files changed, 160 insertions(+), 10 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/LocatorImpl.java b/src/nu/validator/htmlparser/impl/LocatorImpl.java
index 7a559d90..b334aa6a 100644
--- a/src/nu/validator/htmlparser/impl/LocatorImpl.java
+++ b/src/nu/validator/htmlparser/impl/LocatorImpl.java
@@ -24,22 +24,38 @@
 package nu.validator.htmlparser.impl;
 
 import org.xml.sax.Locator;
+import org.xml.sax.ext.Locator2;
 
-public class LocatorImpl implements Locator {
+public class LocatorImpl implements Locator, Locator2 {
 
     private final String systemId;
 
     private final String publicId;
 
+    private final String encoding;
+
     private final int column;
 
     private final int line;
 
+    /**
+     * Create a new Locator with default values
+     */
+    public LocatorImpl() {
+        this.systemId = this.publicId = this.encoding = null;
+        this.column = this.line = 0;
+    }
+
     public LocatorImpl(Locator locator) {
         this.systemId = locator.getSystemId();
         this.publicId = locator.getPublicId();
         this.column = locator.getColumnNumber();
         this.line = locator.getLineNumber();
+        if (locator instanceof Locator2) {
+            this.encoding = ((Locator2)locator).getEncoding();
+        } else {
+            this.encoding = null;
+        }
     }
 
     public final int getColumnNumber() {
@@ -57,4 +73,12 @@ public final String getPublicId() {
     public final String getSystemId() {
         return systemId;
     }
+
+    public final String getXMLVersion() {
+        return "1.0";
+    }
+
+    public final String getEncoding() {
+        return encoding;
+    }
 }
diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index c8e1828a..14f0a832 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -37,6 +37,7 @@
 
 import org.xml.sax.ErrorHandler;
 import org.xml.sax.Locator;
+import org.xml.sax.ext.Locator2;
 import org.xml.sax.SAXException;
 import org.xml.sax.SAXParseException;
 
@@ -66,7 +67,7 @@
  * @version $Id$
  * @author hsivonen
  */
-public class Tokenizer implements Locator {
+public class Tokenizer implements Locator, Locator2 {
 
     private static final int DATA_AND_RCDATA_MASK = ~1;
 
@@ -892,6 +893,24 @@ public String getSystemId() {
         return systemId;
     }
 
+    /**
+     * @see org.xml.sax.ext.Locator2#getXMLVersion()
+     */
+    public String getXMLVersion() {
+        return "1.0";
+    }
+
+    /**
+     * @see org.xml.sax.ext.Locator2#getXMLVersion()
+     */
+    public String getEncoding() {
+        try {
+            return encodingDeclarationHandler == null ? null : encodingDeclarationHandler.getCharacterEncoding();
+        } catch (SAXException e) {
+            return null;
+        }
+    }
+
     // end Locator impl
 
     // end public API
diff --git a/src/nu/validator/htmlparser/io/Driver.java b/src/nu/validator/htmlparser/io/Driver.java
index a3277ea8..d685e69d 100644
--- a/src/nu/validator/htmlparser/io/Driver.java
+++ b/src/nu/validator/htmlparser/io/Driver.java
@@ -588,7 +588,7 @@ public void setXmlnsPolicy(XmlViolationPolicy xmlnsPolicy) {
     }
 
     public String getCharacterEncoding() throws SAXException {
-        return characterEncoding.getCanonName();
+        return characterEncoding == null ? null : characterEncoding.getCanonName();
     }
 
     public Locator getDocumentLocator() {
diff --git a/src/nu/validator/htmlparser/io/HtmlInputStreamReader.java b/src/nu/validator/htmlparser/io/HtmlInputStreamReader.java
index 63884326..3de1af2a 100755
--- a/src/nu/validator/htmlparser/io/HtmlInputStreamReader.java
+++ b/src/nu/validator/htmlparser/io/HtmlInputStreamReader.java
@@ -42,6 +42,7 @@
 
 import org.xml.sax.ErrorHandler;
 import org.xml.sax.Locator;
+import org.xml.sax.ext.Locator2;
 import org.xml.sax.SAXException;
 import org.xml.sax.SAXParseException;
 
@@ -58,7 +59,7 @@
  * @author hsivonen
  */
 public final class HtmlInputStreamReader extends Reader implements
-        ByteReadable, Locator {
+        ByteReadable, Locator, Locator2 {
 
     private static final int SNIFFING_LIMIT = 1024;
 
@@ -452,6 +453,20 @@ public String getSystemId() {
         return null;
     }
 
+    public String getXMLVersion() {
+        if (tokenizer != null) {
+            return tokenizer.getXMLVersion();
+        }
+        return null;
+    }
+
+    public String getEncoding() {
+        if (tokenizer != null) {
+            return tokenizer.getEncoding();
+        }
+        return null;
+    }
+
     /**
      * @param string
      * @throws SAXException
diff --git a/src/nu/validator/htmlparser/io/MetaSniffer.java b/src/nu/validator/htmlparser/io/MetaSniffer.java
index baa04e44..9deaef7a 100755
--- a/src/nu/validator/htmlparser/io/MetaSniffer.java
+++ b/src/nu/validator/htmlparser/io/MetaSniffer.java
@@ -30,10 +30,11 @@
 
 import org.xml.sax.ErrorHandler;
 import org.xml.sax.Locator;
+import org.xml.sax.ext.Locator2;
 import org.xml.sax.SAXException;
 import org.xml.sax.SAXParseException;
 
-public class MetaSniffer extends MetaScanner implements Locator {
+public class MetaSniffer extends MetaScanner implements Locator, Locator2 {
     
     private Encoding characterEncoding = null;
 
@@ -142,6 +143,20 @@ public String getSystemId() {
         }
         return null;
     }
+
+    public String getXMLVersion() {
+        if (locator != null) {
+            return ((Locator2)locator).getXMLVersion();
+        }
+        return null;
+    }
+
+    public String getEncoding() {
+        if (locator != null) {
+            return ((Locator2)locator).getEncoding();
+        }
+        return null;
+    }
     
     protected boolean tryCharset(String encoding) throws SAXException {
         encoding = Encoding.toAsciiLowerCase(encoding);
diff --git a/src/nu/validator/saxtree/DocumentFragment.java b/src/nu/validator/saxtree/DocumentFragment.java
index 06816932..a9e2db84 100644
--- a/src/nu/validator/saxtree/DocumentFragment.java
+++ b/src/nu/validator/saxtree/DocumentFragment.java
@@ -23,7 +23,7 @@
 
 package nu.validator.saxtree;
 
-import org.xml.sax.helpers.LocatorImpl;
+import nu.validator.htmlparser.impl.LocatorImpl;
 
 /**
  * A document fragment.
diff --git a/src/nu/validator/saxtree/LocatorImpl.java b/src/nu/validator/saxtree/LocatorImpl.java
index 37c0c632..b6416f1a 100644
--- a/src/nu/validator/saxtree/LocatorImpl.java
+++ b/src/nu/validator/saxtree/LocatorImpl.java
@@ -24,13 +24,14 @@
 package nu.validator.saxtree;
 
 import org.xml.sax.Locator;
+import org.xml.sax.ext.Locator2;
 
 /**
  * A locator implementation.
  * @version $Id$
  * @author hsivonen
  */
-public final class LocatorImpl implements Locator {
+public final class LocatorImpl implements Locator, Locator2 {
 
     /**
      * The system id.
@@ -42,6 +43,11 @@ public final class LocatorImpl implements Locator {
      */
     private final String publicId;
     
+    /**
+     * The encoding.
+     */
+    private final String encoding;
+    
     /**
      * The column.
      */
@@ -62,11 +68,17 @@ public LocatorImpl(Locator locator) {
             this.publicId = null;
             this.column = -1;
             this.line = -1;
+            this.encoding = null;
         } else {
             this.systemId = locator.getSystemId();
             this.publicId = locator.getPublicId();
             this.column = locator.getColumnNumber();
             this.line = locator.getLineNumber();
+            if (locator instanceof Locator2) {
+                this.encoding = ((Locator2)locator).getEncoding();
+            } else {
+                this.encoding = null;
+            }
         }
     }
     
@@ -101,4 +113,21 @@ public String getPublicId() {
     public String getSystemId() {
         return systemId;
     }
+
+    /**
+     * 
+     * @see org.xml.sax.ext.Locator2#getXMLVersion()
+     */
+    public String getXMLVersion() {
+        return "1.0";
+    }
+
+    /**
+     * 
+     * @see org.xml.sax.ext.Locator2#getEncoding()
+     */
+    public String getEncoding() {
+        return encoding;
+    }
+
 }
diff --git a/src/nu/validator/saxtree/Node.java b/src/nu/validator/saxtree/Node.java
index 7aed83b7..c9292dd2 100644
--- a/src/nu/validator/saxtree/Node.java
+++ b/src/nu/validator/saxtree/Node.java
@@ -27,6 +27,7 @@
 
 import org.xml.sax.Attributes;
 import org.xml.sax.Locator;
+import org.xml.sax.ext.Locator2;
 import org.xml.sax.SAXException;
 
 /**
@@ -34,7 +35,7 @@
  * @version $Id$
  * @author hsivonen
  */
-public abstract class Node implements Locator {
+public abstract class Node implements Locator, Locator2 {
 
     /**
      * The system id.
@@ -45,6 +46,11 @@ public abstract class Node implements Locator {
      * The public id.
      */
     private final String publicId;
+
+    /**
+     * The encoding.
+     */
+    private final String encoding;
     
     /**
      * The column.
@@ -75,6 +81,7 @@ public abstract class Node implements Locator {
         if (locator == null) {
             this.systemId = null;
             this.publicId = null;
+            this.encoding = null;
             this.column = -1;
             this.line = -1;
         } else {
@@ -82,6 +89,11 @@ public abstract class Node implements Locator {
             this.publicId = locator.getPublicId();
             this.column = locator.getColumnNumber();
             this.line = locator.getLineNumber();
+            if (locator instanceof Locator2) {
+                this.encoding = ((Locator2)locator).getEncoding();
+            } else {
+                this.encoding = null;
+            }
         }
     }
     
@@ -117,6 +129,20 @@ public String getSystemId() {
         return systemId;
     }
 
+    /**
+     * @see org.xml.sax.ext.Locator2#getXMLVersion()
+     */
+    public String getXMLVersion() {
+        return "1.0";
+    }
+
+    /**
+     * @see org.xml.sax.ext.Locator2#getEncoding
+     */
+    public String getEncoding() {
+        return encoding;
+    }
+
     /**
      * Visit the node.
      * 
diff --git a/src/nu/validator/saxtree/TreeParser.java b/src/nu/validator/saxtree/TreeParser.java
index a9d92deb..6f86f7f0 100644
--- a/src/nu/validator/saxtree/TreeParser.java
+++ b/src/nu/validator/saxtree/TreeParser.java
@@ -26,6 +26,7 @@
 import org.xml.sax.Attributes;
 import org.xml.sax.ContentHandler;
 import org.xml.sax.Locator;
+import org.xml.sax.ext.Locator2;
 import org.xml.sax.SAXException;
 import org.xml.sax.ext.LexicalHandler;
 
@@ -34,7 +35,7 @@
  * @version $Id$
  * @author hsivonen
  */
-public final class TreeParser implements Locator {
+public final class TreeParser implements Locator, Locator2 {
     
     /**
      * The content handler.
@@ -283,7 +284,6 @@ public String getPublicId() {
         if (locatorDelegate == null) {
             return null;
         } else {
-
             return locatorDelegate.getPublicId();
         }
     }
@@ -298,4 +298,26 @@ public String getSystemId() {
             return locatorDelegate.getSystemId();
         }
     }
+
+    /**
+     * @see org.xml.sax.Locator#getSystemId()
+     */
+    public String getXMLVersion() {
+        if (!(locatorDelegate instanceof Locator2)) {
+            return null;
+        } else {
+            return ((Locator2)locatorDelegate).getXMLVersion();
+        }
+    }
+
+    /**
+     * @see org.xml.sax.Locator#getSystemId()
+     */
+    public String getEncoding() {
+        if (!(locatorDelegate instanceof Locator2)) {
+            return null;
+        } else {
+            return ((Locator2)locatorDelegate).getEncoding();
+        }
+    }
 }

From 9e152516af38809a35743fdb35a0e50dff99a4de Mon Sep 17 00:00:00 2001
From: Henri Sivonen <hsivonen@hsivonen.fi>
Date: Thu, 2 Jul 2020 15:56:47 +0300
Subject: [PATCH 16/18] Mozilla bug 1650066 fixup - Avoid calling
 errListUnclosedStartTags in C++.

Differential Revision: https://phabricator.services.mozilla.com/D82007
---
 src/nu/validator/htmlparser/impl/TreeBuilder.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/nu/validator/htmlparser/impl/TreeBuilder.java b/src/nu/validator/htmlparser/impl/TreeBuilder.java
index 860f6f5b..02cc6409 100644
--- a/src/nu/validator/htmlparser/impl/TreeBuilder.java
+++ b/src/nu/validator/htmlparser/impl/TreeBuilder.java
@@ -1520,9 +1520,11 @@ public final void eof() throws SAXException {
                         assert fragment;
                         break eofloop;
                     }
+                    // [NOCPP[
                     if (errorHandler != null) {
                         errListUnclosedStartTags(0);
                     }
+                    // ]NOCPP]
                     while (currentPtr >= eltPos) {
                         pop();
                     }

From c7a2fced29f04023de00dc5f1edbbf374eddcc2f Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Thu, 2 Jul 2020 16:07:24 +0900
Subject: [PATCH 17/18] Stay in the COMMENT_LESSTHAN state, annotate
 fall-throughs.

---
 src/nu/validator/htmlparser/impl/Tokenizer.java | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index 14f0a832..a9604a14 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -2924,8 +2924,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue stateloop;
                             case '<':
                                 appendStrBuf(c);
-                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
-                                continue stateloop;
+                                continue;
                             case '-':
                                 appendStrBuf(c);
                                 state = transition(state, Tokenizer.COMMENT_END_DASH, reconsume, pos);
@@ -2938,13 +2937,14 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue;
                             case '\u0000':
                                 c = '\uFFFD';
-                                // fall thru
+                                // CPPONLY: MOZ_FALLTHROUGH;
                             default:
                                 appendStrBuf(c);
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
                                 continue stateloop;
                         }
                     }
+                    // CPPONLY: MOZ_FALLTHROUGH;
                 case COMMENT_LESSTHAN_BANG:
                     for (;;) {
                         if (++pos == endPos) {
@@ -2968,13 +2968,14 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue;
                             case '\u0000':
                                 c = '\uFFFD';
-                                // fall thru
+                                // CPPONLY: MOZ_FALLTHROUGH;
                             default:
                                 appendStrBuf(c);
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
                                 continue stateloop;
                         }
                     }
+                    // CPPONLY: MOZ_FALLTHROUGH;
                 case COMMENT_LESSTHAN_BANG_DASH:
                     for (;;) {
                         if (++pos == endPos) {
@@ -2998,13 +2999,14 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue;
                             case '\u0000':
                                 c = '\uFFFD';
-                                // fall thru
+                                // CPPONLY: MOZ_FALLTHROUGH;
                             default:
                                 appendStrBuf(c);
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
                                 continue stateloop;
                         }
                     }
+                    // CPPONLY: MOZ_FALLTHROUGH;
                 case COMMENT_LESSTHAN_BANG_DASH_DASH:
                     for (;;) {
                         if (++pos == endPos) {
@@ -3037,7 +3039,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue;
                             case '\u0000':
                                 c = '\uFFFD';
-                                // fall thru
+                                // CPPONLY: MOZ_FALLTHROUGH;
                             case '!':
                                 errNestedComment();
                                 adjustDoubleHyphenAndAppendToStrBufAndErr(c, reportedConsecutiveHyphens);
@@ -3052,6 +3054,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue stateloop;
                         }
                     }
+                    // CPPONLY: MOZ_FALLTHROUGH;
                     // XXX reorder point
                 case COMMENT_START_DASH:
                     if (++pos == endPos) {

From 8010453bda62790f4b6a80958270714a9e0bad34 Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Thu, 2 Jul 2020 20:41:36 +0900
Subject: [PATCH 18/18] Limit ambiguous-ampersand wasteful buffering
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This refines 9ce4bd4 by only buffering if we’re actually inside an
attribute value.
---
 src/nu/validator/htmlparser/impl/Tokenizer.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index a9604a14..45073f85 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -3610,7 +3610,13 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                         } else if ((c >= '0' && c <= '9')
                                 || (c >= 'A' && c <= 'Z')
                                 || (c >= 'a' && c <= 'z')) {
-                            appendStrBuf(c);
+                            if (returnState == ATTRIBUTE_VALUE_DOUBLE_QUOTED
+                                    || returnState == ATTRIBUTE_VALUE_SINGLE_QUOTED
+                                    || returnState == ATTRIBUTE_VALUE_UNQUOTED) {
+                                appendStrBuf(c);
+                            }
+                            /* The following pos++ is necessary due to how we’ve
+                             * handled the "reconsume" block for this case. */
                             pos++;
                             continue;
                         }