From 549df0507c2d2fcb8e3136611ebfe63a4136b952 Mon Sep 17 00:00:00 2001 From: Parker Loflin Date: Mon, 2 Nov 2026 10:05:42 -0500 Subject: [PATCH] Speed up CGI.escapeHTML with an SSE2/NEON search CGI.escapeHTML checked one byte at a time and always allocated a buffer six times the input's size, even when nothing needed escaping. ERB::Util.html_escape does the same job and got a SIMD search in ruby/erb#136. Port erb's SSE2/NEON match code (originally from the json gem). Strings of 64 bytes or more are searched for the first ' & < > ", and a string with none is returned by rb_str_dup with no buffer. Otherwise the clean prefix is copied at once, clean 16-byte blocks are copied whole, and the other bytes are escaped 4 at a time. Strings under 64 bytes skip the search, since it costs more than it saves when they need escaping. The 4-byte step replaces a byte loop because gcc and clang each compiled some forms of that loop into code slower than the old one. escape_html_into is NOINLINE so that with _FORTIFY_SOURCE=3 its small memcpy calls stay plain moves instead of becoming __memcpy_chk calls. Without SIMD (CGI_DISABLE_SIMD, MSVC, other CPUs) the search checks 8 bytes per step and is skipped under 128 bytes. extconf.rb uses erb's compile check to define CGI_ENABLE_SIMD. Output, encoding and the "always a new, unfrozen String" behaviour are unchanged. --- ext/cgi/escape/escape.c | 432 ++++++++++++++++++++++++++++++++++-- ext/cgi/escape/extconf.rb | 26 +++ test/cgi/test_cgi_escape.rb | 58 +++++ 3 files changed, 499 insertions(+), 17 deletions(-) diff --git a/ext/cgi/escape/escape.c b/ext/cgi/escape/escape.c index 5d787d0..026471e 100644 --- a/ext/cgi/escape/escape.c +++ b/ext/cgi/escape/escape.c @@ -55,35 +55,433 @@ escaped_length(VALUE str) return len * HTML_ESCAPE_MAX_LEN; } +/* + * Fast path for optimized_escape_html. + * + * Strings under HTML_ESCAPE_ONE_PASS bytes are escaped in one pass into a + * stack buffer. Longer strings are searched for the first byte to escape, + * and a string with none is returned by rb_str_dup. Otherwise + * escape_html_into copies the clean prefix at once and escapes the rest. + * + * The SSE2 and NEON code is ported from ruby/erb's ext/erb/escape/escape.c + * (ruby/erb#136 and follow-ups), which took it from the json gem. Set + * CGI_DISABLE_SIMD when building to leave it out. + */ + +#ifdef __clang__ +# if __has_builtin(__builtin_ctzll) +# define HAVE_BUILTIN_CTZLL 1 +# else +# define HAVE_BUILTIN_CTZLL 0 +# endif +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +# define HAVE_BUILTIN_CTZLL 1 +#else +# define HAVE_BUILTIN_CTZLL 0 +#endif + +#ifdef CGI_ENABLE_SIMD +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) +#ifdef HAVE_X86INTRIN_H +#include +#define HAVE_SIMD 1 +#define HAVE_SIMD_SSE2 1 +#endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#define HAVE_SIMD 1 +#define HAVE_SIMD_NEON 1 +#include +#endif +#endif /* CGI_ENABLE_SIMD */ + +/* + * Strings shorter than this skip the search and are escaped in one pass + * into a stack buffer. With SIMD, the search pays off on longer strings + * that need no escaping. Without SIMD it costs about as much as escaping, + * so the limit only sizes the stack buffer, which stays under the 1 KB that + * ALLOCV already put on the stack. + */ +#ifdef HAVE_SIMD +# define HTML_ESCAPE_ONE_PASS 64 +#else +# define HTML_ESCAPE_ONE_PASS 128 +#endif + +#ifdef HAVE_SIMD +static inline unsigned int +trailing_zeros64(uint64_t input) +{ +#if HAVE_BUILTIN_CTZLL + return __builtin_ctzll(input); +#else + unsigned int trailing_zeros = 0; + uint64_t temp = input; + while ((temp & 1) == 0 && temp > 0) { + trailing_zeros++; + temp >>= 1; + } + return trailing_zeros; +#endif +} +#endif + +#ifdef HAVE_SIMD_SSE2 +/* Bit i is set if p[i] is one of ' " & < > */ +static inline uint64_t +escape_bitmap16(const unsigned char *p) +{ + const __m128i bytes = _mm_loadu_si128((__m128i const *)p); + const __m128i match1 = _mm_cmpeq_epi8(bytes, _mm_set1_epi8('\'')); + const __m128i match2 = _mm_cmpeq_epi8(bytes, _mm_set1_epi8('"')); + const __m128i match3 = _mm_cmpeq_epi8(bytes, _mm_set1_epi8('&')); + const __m128i match4 = _mm_cmpeq_epi8(bytes, _mm_set1_epi8('<')); + const __m128i match5 = _mm_cmpeq_epi8(bytes, _mm_set1_epi8('>')); + + const __m128i mask1 = _mm_or_si128(match1, match2); + const __m128i mask2 = _mm_or_si128(match3, match4); + const __m128i mask3 = _mm_or_si128(mask1, match5); + const __m128i matches = _mm_or_si128(mask2, mask3); + + return (unsigned int)_mm_movemask_epi8(matches); +} +#endif /* HAVE_SIMD_SSE2 */ + +#ifdef HAVE_SIMD_NEON +/* + * The escapable character with each low nibble, if any. Index 0 holds + * '\'' instead of 0 so that NUL bytes do not match. + */ +static const uint8x16_t escape_char_by_low_nibble = { + '\'', 0, '"', 0, + 0, 0, '&', '\'', + 0, 0, 0, 0, + '<', 0, '>', 0, +}; + +/* + * Returns 0xFF in each lane whose byte needs escaping and 0x00 in the rest, + * by looking up the character for each byte's low nibble and comparing. + * For "
" (3c 62 72 20 2f 3e), the low nibbles (0c 02 02 00 0f 0e) + * look up (3c 22 22 27 00 3e), so only the first and last lanes match. + */ +static inline uint8x16_t +neon_escape_matches(const uint8x16_t bytes) +{ + const uint8x16_t low_nibbles = vandq_u8(bytes, vdupq_n_u8(0x0F)); + const uint8x16_t looked_up = vqtbl1q_u8(escape_char_by_low_nibble, low_nibbles); + return vceqq_u8(looked_up, bytes); +} + +static inline uint64_t +neon_matches_to_bitmap16(const uint8x16_t matches) +{ + static const uint8x16_t bit_mask = { + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + }; + + uint8x16_t folded = vandq_u8(matches, bit_mask); + folded = vpaddq_u8(folded, folded); + folded = vpaddq_u8(folded, folded); + folded = vpaddq_u8(folded, folded); + + return vgetq_lane_u16(vreinterpretq_u16_u8(folded), 0); +} + +static inline uint64_t +neon_matches_to_bitmap64(const uint8x16_t m0, const uint8x16_t m1, const uint8x16_t m2, const uint8x16_t m3) +{ + static const uint8x16_t bit_mask = { + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + }; + + const uint8x16_t t0 = vandq_u8(m0, bit_mask); + const uint8x16_t t1 = vandq_u8(m1, bit_mask); + const uint8x16_t t2 = vandq_u8(m2, bit_mask); + const uint8x16_t t3 = vandq_u8(m3, bit_mask); + + uint8x16_t folded = vpaddq_u8(vpaddq_u8(t0, t1), vpaddq_u8(t2, t3)); + folded = vpaddq_u8(folded, folded); + + return vgetq_lane_u64(vreinterpretq_u64_u8(folded), 0); +} + +/* Bit i is set if p[i] is one of ' " & < > */ +static inline uint64_t +escape_bitmap16(const unsigned char *p) +{ + return neon_matches_to_bitmap16(neon_escape_matches(vld1q_u8(p))); +} +#endif /* HAVE_SIMD_NEON */ + +/* Returns the first byte in [p, end) that needs escaping, or end. */ +static inline const unsigned char * +find_first_escape(const unsigned char *p, const unsigned char *end) +{ +#ifdef HAVE_SIMD + if (end - p >= 16) { + const unsigned char *const last = end - 16; + uint64_t bitmap; + + /* Check the first 16 bytes alone, since escapes often come early. */ + bitmap = escape_bitmap16(p); + if (bitmap) { + return p + trailing_zeros64(bitmap); + } + p += 16; + +# ifdef HAVE_SIMD_NEON + while (end - p >= 64) { + const uint8x16_t m0 = neon_escape_matches(vld1q_u8(p + 0)); + const uint8x16_t m1 = neon_escape_matches(vld1q_u8(p + 16)); + const uint8x16_t m2 = neon_escape_matches(vld1q_u8(p + 32)); + const uint8x16_t m3 = neon_escape_matches(vld1q_u8(p + 48)); + bitmap = neon_matches_to_bitmap64(m0, m1, m2, m3); + + if (bitmap) { + return p + trailing_zeros64(bitmap); + } + p += 64; + } +# endif +# ifdef HAVE_SIMD_SSE2 + while (end - p >= 64) { + bitmap = escape_bitmap16(p) | escape_bitmap16(p + 16) << 16 | + escape_bitmap16(p + 32) << 32 | escape_bitmap16(p + 48) << 48; + if (bitmap) { + return p + trailing_zeros64(bitmap); + } + p += 64; + } +# endif + while (p < last) { + bitmap = escape_bitmap16(p); + if (bitmap) { + return p + trailing_zeros64(bitmap); + } + p += 16; + } + /* The last 16 bytes. Any that overlap the blocks above are clean. */ + bitmap = escape_bitmap16(last); + return bitmap ? last + trailing_zeros64(bitmap) : end; + } +#endif + /* Unrolled by hand to one branch per 8 bytes. */ + while (end - p >= 8) { + if (html_escape_table[p[0]].len | html_escape_table[p[1]].len | + html_escape_table[p[2]].len | html_escape_table[p[3]].len | + html_escape_table[p[4]].len | html_escape_table[p[5]].len | + html_escape_table[p[6]].len | html_escape_table[p[7]].len) { + break; + } + p += 8; + } + while (p < end && !html_escape_table[*p].len) { + p++; + } + return p; +} + +/* + * Appends c, whose escaped length is len (0 if c is clean). The whole + * HTML_ESCAPE_MAX_LEN-byte entry is copied in a couple of moves and dest + * advances by len, so dest needs room for HTML_ESCAPE_MAX_LEN bytes. + */ +#define APPEND_ESCAPED(dest, c, len) do { \ + if (len) { \ + memcpy((dest), html_escape_table[c].str, HTML_ESCAPE_MAX_LEN); \ + (dest) += (len); \ + } \ + else { \ + *(dest)++ = (char)(c); \ + } \ +} while (0) + +/* + * Appends the 4 bytes at p: in one move if all 4 are clean, else one at a + * time with the lengths already looked up. dest needs room for + * 4 * HTML_ESCAPE_MAX_LEN bytes. gcc and clang each compiled some + * byte-by-byte loops into code slower than the old loop; this form is at + * least as fast with both. + */ +#define APPEND_ESCAPED4(dest, p) do { \ + const unsigned char c0_ = (p)[0], c1_ = (p)[1]; \ + const unsigned char c2_ = (p)[2], c3_ = (p)[3]; \ + const uint8_t len0_ = html_escape_table[c0_].len; \ + const uint8_t len1_ = html_escape_table[c1_].len; \ + const uint8_t len2_ = html_escape_table[c2_].len; \ + const uint8_t len3_ = html_escape_table[c3_].len; \ + if (!(len0_ | len1_ | len2_ | len3_)) { \ + memcpy((dest), (p), 4); \ + (dest) += 4; \ + } \ + else { \ + APPEND_ESCAPED((dest), c0_, len0_); \ + APPEND_ESCAPED((dest), c1_, len1_); \ + APPEND_ESCAPED((dest), c2_, len2_); \ + APPEND_ESCAPED((dest), c3_, len3_); \ + } \ +} while (0) + +/* + * Escapes [start, end) into dest and returns the end of the output. The + * bytes in [start, first) are clean, and first > start only for strings of + * 16 bytes or more. dest needs room for HTML_ESCAPE_MAX_LEN bytes per + * input byte. + * + * NOINLINE because, inlined into optimized_escape_html, the compiler sees + * the buffer's size, and with _FORTIFY_SOURCE=3 (the default on some + * distributions) the small fixed-size memcpy calls below would become + * __memcpy_chk calls. + */ +NOINLINE(static char *escape_html_into(char *dest, const unsigned char *start, const unsigned char *first, const unsigned char *end)); + +static char * +escape_html_into(char *dest, const unsigned char *start, const unsigned char *first, const unsigned char *end) +{ + const unsigned char *cstr = first; + const size_t prefix = first - start; + + /* + * Copy the clean prefix: one memcpy call if it is long, else 16-byte + * moves, which cost less than a call. The last move ends at first. A + * prefix under 16 bytes is copied as 16, which the input has (see + * above) and dest has room for, and the extra bytes are overwritten. + */ + if (prefix >= 256) { + memcpy(dest, start, prefix); + } + else if (prefix >= 16) { + size_t i; + + for (i = 0; i + 16 < prefix; i += 16) { + memcpy(dest + i, start + i, 16); + } + memcpy(dest + prefix - 16, first - 16, 16); + } + else if (prefix) { + memcpy(dest, start, 16); + } + dest += prefix; + +#ifdef HAVE_SIMD + { + /* + * While 64 or more bytes are left, take 16 at a time: copy a clean + * block whole and escape any other 4 bytes at a time. After a block + * with 5 or more escapes, escape the next one without searching it, + * since in text that dense a clean block is rare. On + * the last 63 bytes or fewer, the search costs more than it saves. + */ + int dense = 0; + + while (end - cstr >= 64) { + if (!dense && !escape_bitmap16(cstr)) { + memcpy(dest, cstr, 16); + dest += 16; + } + else { + char *const block = dest; + + APPEND_ESCAPED4(dest, cstr); + APPEND_ESCAPED4(dest, cstr + 4); + APPEND_ESCAPED4(dest, cstr + 8); + APPEND_ESCAPED4(dest, cstr + 12); + /* Five or more escapes add at least 15 bytes. */ + dense = dest - block >= 16 + 15; + } + cstr += 16; + } + } +#endif + + /* The rest 4 bytes at a time, then the last 0 to 3 bytes. */ + while (end - cstr >= 4) { + APPEND_ESCAPED4(dest, cstr); + cstr += 4; + } + while (cstr < end) { + APPEND_ESCAPED(dest, *cstr, html_escape_table[*cstr].len); + cstr++; + } + return dest; +} + static VALUE optimized_escape_html(VALUE str) { VALUE escaped; VALUE vbuf; - char *buf = ALLOCV_N(char, vbuf, escaped_length(str)); - const char *cstr = RSTRING_PTR(str); - const char *end = cstr + RSTRING_LEN(str); - - char *dest = buf; - while (cstr < end) { - const unsigned char c = *cstr++; - uint8_t len = html_escape_table[c].len; - if (len) { - memcpy(dest, html_escape_table[c].str, len); - dest += len; + char *buf, *dest; + const unsigned char *start = (const unsigned char *)RSTRING_PTR(str); + const unsigned char *end = start + RSTRING_LEN(str); + const unsigned char *first; + + if (end - start < 16) { + /* + * Strings under 16 bytes: the old loop, into a stack buffer. The + * escapes are copied byte by byte because a memcpy into sbuf would + * become a __memcpy_chk call with _FORTIFY_SOURCE=3. + */ + char sbuf[15 * HTML_ESCAPE_MAX_LEN]; + const unsigned char *p; + + dest = sbuf; + for (p = start; p < end; p++) { + const unsigned char c = *p; + const uint8_t len = html_escape_table[c].len; + + if (len) { + const char *const e = html_escape_table[c].str; + + dest[0] = e[0]; + dest[1] = e[1]; + dest[2] = e[2]; + dest[3] = e[3]; + dest[4] = e[4]; + dest[5] = e[5]; + dest += len; + } + else { + *dest++ = (char)c; + } } - else { - *dest++ = c; + if (dest - sbuf == end - start) { + return rb_str_dup(str); } + escaped = rb_str_new(sbuf, dest - sbuf); + preserve_original_state(str, escaped); + return escaped; } - if (RSTRING_LEN(str) < (dest - buf)) { - escaped = rb_str_new(buf, dest - buf); + if (end - start < HTML_ESCAPE_ONE_PASS) { + /* + * Short strings: escape in one pass into a stack buffer, since + * searching first would read them twice. + */ + char sbuf[(HTML_ESCAPE_ONE_PASS - 1) * HTML_ESCAPE_MAX_LEN]; + + dest = escape_html_into(sbuf, start, start, end); + if (dest - sbuf == end - start) { + return rb_str_dup(str); + } + escaped = rb_str_new(sbuf, dest - sbuf); preserve_original_state(str, escaped); + return escaped; } - else { - escaped = rb_str_dup(str); + + first = find_first_escape(start, end); + if (first == end) { + return rb_str_dup(str); } + + buf = ALLOCV_N(char, vbuf, escaped_length(str)); + dest = escape_html_into(buf, start, first, end); + escaped = rb_str_new(buf, dest - buf); + preserve_original_state(str, escaped); ALLOCV_END(vbuf); return escaped; } diff --git a/ext/cgi/escape/extconf.rb b/ext/cgi/escape/extconf.rb index db4c5ca..ec124cc 100644 --- a/ext/cgi/escape/extconf.rb +++ b/ext/cgi/escape/extconf.rb @@ -10,5 +10,31 @@ end have_var(%[ruby_hexdigits[0]], "ruby/util.h") have_var(%[ruby_digit36_to_number_table[0]], "ruby/util.h") + + # Same SIMD detection as ruby/erb's ext/erb/escape/extconf.rb. + # Set CGI_DISABLE_SIMD to build the scalar path only. + unless ENV["CGI_DISABLE_SIMD"] + case RbConfig::CONFIG['host_cpu'] + when /^(arm64|aarch64)/ + # Try to compile a small program using NEON instructions + header, type, init, extra = 'arm_neon.h', 'uint8x16_t', 'vdupq_n_u8(32)', nil + when /^(x86_64|x64)/ + header, type, init, extra = 'x86intrin.h', '__m128i', '_mm_set1_epi8(32)', 'if (__builtin_cpu_supports("sse2")) { printf("OK"); }' + end + if header + if have_header(header) && try_compile(<<~SRC, '-Werror=implicit-function-declaration') + #{cpp_include(header)} + int main(int argc, char **argv) { + #{type} test = #{init}; + #{extra} + if (argc > 100000) printf("%p", &test); + return 0; + } + SRC + $defs.push("-DCGI_ENABLE_SIMD") + end + end + end + create_makefile 'cgi/escape' end diff --git a/test/cgi/test_cgi_escape.rb b/test/cgi/test_cgi_escape.rb index 3278bde..05c935c 100644 --- a/test/cgi/test_cgi_escape.rb +++ b/test/cgi/test_cgi_escape.rb @@ -177,6 +177,64 @@ def test_cgi_escape_html_dont_freeze assert_not_predicate CGI.escapeHTML("Ruby".freeze), :frozen? end + HTML_ESCAPES = { "'" => "'", "&" => "&", '"' => """, "<" => "<", ">" => ">" }.freeze + + # Each escapable character at every position, for lengths around the + # 16-, 32- and 64-byte steps of the SIMD code. + def test_cgi_escape_html_every_position + [*0..80, 127, 128, 129].each do |len| + assert_equal("a" * len, CGI.escapeHTML("a" * len)) + len.times do |i| + HTML_ESCAPES.each do |char, escaped| + str = "a" * len + str[i] = char + assert_equal("a" * i + escaped + "a" * (len - i - 1), CGI.escapeHTML(str), "len=#{len} i=#{i} char=#{char}") + end + end + end + end + + # Every other byte value is kept as is, including those that share a low + # nibble or the low 7 bits with an escapable character. + def test_cgi_escape_html_other_bytes + (0..255).each do |byte| + str = ("x" * 40).b + str.setbyte(20, byte) + expected = HTML_ESCAPES.fetch(byte.chr, byte.chr.b) + assert_equal(("x" * 20).b + expected.b + ("x" * 19).b, CGI.escapeHTML(str).b, "byte=#{byte}") + end + end + + # Every k-th byte needs escaping, from every 8th byte to every byte. + def test_cgi_escape_html_dense + specials = HTML_ESCAPES.keys + [*0..80, 127, 128, 129, 255, 256, 257].each do |len| + [1, 2, 3, 4, 8].each do |k| + str = Array.new(len) { |i| i % k == 0 ? specials[i % specials.size] : "a" }.join + assert_equal(str.gsub(/['&"<>]/, HTML_ESCAPES), CGI.escapeHTML(str), "len=#{len} k=#{k}") + end + end + end + + # Clean prefixes of 15 to 1000 bytes before the first byte to escape. + def test_cgi_escape_html_clean_prefix + [15, 16, 17, 31, 32, 33, 255, 256, 257, 1000].each do |len| + str = "a" * len + "&\"'" + "c" * 40 + assert_equal("a" * len + "<b>&"'" + "c" * 40, CGI.escapeHTML(str), "len=#{len}") + end + end + + # Strings that start at every offset inside a shared buffer. + def test_cgi_escape_html_unaligned + base = "0123456789&def'ghi\"jkl" * 10 + 32.times do |offset| + [0, 1, 15, 16, 17, 63, 64, 65, 200].each do |len| + str = base.byteslice(offset, len) + assert_equal(str.gsub(/['&"<>]/, HTML_ESCAPES), CGI.escapeHTML(str), "offset=#{offset} len=#{len}") + end + end + end + def test_cgi_escape_html_nil assert_raise(TypeError) { CGI.escapeHTML(nil) } end