Fix SEGV when as_json returns a non-String for an invalid symbol

In strict mode, a Symbol whose name is not UTF-8 compatible was passed to
generate_json_string, which handed the as_json result to
raw_generate_json_string without checking its type. A non-String result was
then read as a String, segfaulting the process.

Route strict symbols through the T_STRING case of generate_json_general so
they reuse its as_json handling. That re-dispatches the result by type, and
its as_json_called flag stops a symbol returned by as_json from recursing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: 2 people

ext/json/ext/generator/generator.c

@@ -1124,15 +1124,6 @@ static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *d
     }
 }
 
-static inline void generate_json_symbol(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
-{
-    if (data->state->strict) {
-        generate_json_string(buffer, data, rb_sym2str(obj));
-    } else {
-        generate_json_fallback(buffer, data, obj);
-    }
-}
-
 static void generate_json_null(FBuffer *buffer, struct generate_json_data *data, VALUE obj)
 {
     fbuffer_append(buffer, "null", 4);
@@ -1218,7 +1209,13 @@ static inline void generate_json_general(FBuffer *buffer, struct generate_json_d
         } else if (RB_FLONUM_P(obj)) {
             generate_json_float(buffer, data, obj);
         } else if (RB_STATIC_SYM_P(obj)) {
-            generate_json_symbol(buffer, data, obj);
+            if (data->state->strict) {
+                obj = rb_sym2str(obj);
+                JSON_ASSERT(RBASIC_CLASS(obj) == rb_cString);
+                goto generate_string;
+            }
+
+            generate_json_fallback(buffer, data, obj);
         } else {
             goto general;
         }
@@ -1239,6 +1236,7 @@ static inline void generate_json_general(FBuffer *buffer, struct generate_json_d
             case T_STRING:
                 if (fallback && klass != rb_cString) goto general;
 
+            generate_string:
                 if (RB_LIKELY(valid_json_string_p(obj))) {
                     raw_generate_json_string(buffer, data, obj);
                 } else if (as_json_called) {
@@ -1250,7 +1248,13 @@ static inline void generate_json_general(FBuffer *buffer, struct generate_json_d
                 }
                 break;
             case T_SYMBOL:
-                generate_json_symbol(buffer, data, obj);
+                if (data->state->strict) {
+                    obj = rb_sym2str(obj);
+                    JSON_ASSERT(RBASIC_CLASS(obj) == rb_cString);
+                    goto generate_string;
+                }
+
+                generate_json_fallback(buffer, data, obj);
                 break;
             case T_FLOAT:
                 if (fallback && klass != rb_cFloat) goto general;

test/json/json_coder_test.rb

@@ -137,6 +137,33 @@ def test_json_coder_string_invalid_encoding
     assert_equal 2, calls
   end
 
+  def test_json_coder_symbol_invalid_encoding
+    symbol = "\xFF".b.to_sym
+
+    calls = 0
+    coder = JSON::Coder.new do |object, is_key|
+      calls += 1
+      object.bytes
+    end
+
+    assert_equal "[[255]]", coder.dump([symbol])
+    assert_equal 1, calls
+
+    coder = JSON::Coder.new { |object, is_key| nil }
+    assert_equal "[null]", coder.dump([symbol])
+
+    coder = JSON::Coder.new { |object, is_key| Object.new }
+    assert_raise(JSON::GeneratorError) { coder.dump([symbol]) }
+
+    if RUBY_ENGINE == "ruby"
+      coder = JSON::Coder.new { |object, is_key| symbol }
+      error = assert_raise JSON::GeneratorError do
+        coder.dump([symbol])
+      end
+      assert_equal "source sequence is illegal/malformed utf-8", error.message
+    end
+  end
+
   def test_depth
     coder = JSON::Coder.new(object_nl: "\n", array_nl: "\n", space: " ", indent: "  ", depth: 1)
     assert_equal %({\n    "foo": 42\n  }), coder.dump(foo: 42)