Make csv write defaults better formatted for opening in excel

Author: nh43de

src/DataPowerTools.Connectivity/Json/DataReaderJsonExtensions.cs

@@ -65,19 +65,28 @@ public static string ToJsonCurrentRecord(this IDataRecord record, bool indent =
         }
 
         /// <summary>
-        /// Creates insert statements from an array of json objects.
+        /// Converts JSON array to CSV string with Excel-friendly formatting.
+        /// 
+        /// Excel Compatibility Notes:
+        /// - Uses UTF-8 compatible formatting with RFC 4180 compliance
+        /// - String output doesn't include BOM - for Excel with emojis, save string to file with UTF-8 BOM
         /// </summary>
-        public static string FromJsonToCsv(this string jsonString, bool writeHeaders = true, bool useTabFormat = false)
+        /// <param name="jsonString">The JSON array string to convert</param>
+        /// <param name="writeHeaders">Whether to write headers</param>
+        /// <param name="useTabFormat">Whether to use tab format (TSV)</param>
+        /// <param name="format">The format to use for the CSV output</param>
+        /// <returns>CSV formatted string</returns>
+        public static string FromJsonToCsv(this string jsonString, bool writeHeaders = true, bool useTabFormat = false, CSVFormat format = CSVFormat.UTF8)
         {
             var sb = new StringBuilder();
-            
+
             var el = JsonDocument.Parse(jsonString).RootElement;
 
             var sw = new StringWriter(sb);
 
             using var csvWriter = useTabFormat
-                ? new CSVWriter(sw, '\t', CSVWriter.DefaultQuoteCharacter, CSVWriter.DefaultEscapeCharacter, "\r\n") 
-                : new CSVWriter(sw,',', CSVWriter.DefaultQuoteCharacter, CSVWriter.DefaultEscapeCharacter, "\r\n");
+                ? new CSVWriter(sw, '\t', CSVWriter.DefaultQuoteCharacter, CSVWriter.DefaultEscapeCharacter, CSVWriter.Rfc4180LineEnd) 
+                : new CSVWriter(sw,',', CSVWriter.DefaultQuoteCharacter, CSVWriter.DefaultEscapeCharacter, CSVWriter.Rfc4180LineEnd);
 
             var hashSetHeaders = new HashSet<string>();
 

src/DataPowerTools.Tests/CsvTests/DataPowerToolsCsvTests.cs

@@ -98,5 +98,100 @@ public void TestCsvExportUsesWindowsLineEndings()
             if (System.IO.File.Exists(testFile))
                 System.IO.File.Delete(testFile);
         }
+
+        [TestMethod]
+        public void TestCsvFormatOptions()
+        {
+            var testData = new[]
+            {
+                new { Name = "Test×Data", Description = "Special characters: ñ©€Á" },
+                new { Name = "Normal", Description = "Regular text" }
+            };
+
+            // Test UTF8 format (now default)
+            var csvContent = testData.ToCsvString(format: CSVFormat.UTF8);
+            Assert.IsTrue(csvContent.Contains("×"));
+            Assert.IsTrue(csvContent.Contains("ñ"));
+            Assert.IsTrue(csvContent.Contains("©"));
+            Assert.IsTrue(csvContent.Contains("€"));
+            Assert.IsTrue(csvContent.Contains("Á"));
+
+            // Test ANSI format (should work with fallback to UTF-8)
+            var csvContentAnsi = testData.ToCsvString(format: CSVFormat.ANSI);
+            Assert.IsTrue(csvContentAnsi.Contains("×"));
+
+            // Test UTF16 format
+            var csvContentUtf16 = testData.ToCsvString(format: CSVFormat.UTF16);
+            Assert.IsTrue(csvContentUtf16.Contains("×"));
+            Assert.IsTrue(csvContentUtf16.Contains("€"));
+        }
+
+        [TestMethod]
+        public void TestCsvEmojiSupportAndExcelCompatibility()
+        {
+            // Test data with emojis and special characters that require full Unicode support
+            var testData = new[]
+            {
+                new { Name = "John 😀 Doe", Status = "Happy 🎉", Country = "🇺🇸 USA" },
+                new { Name = "María José 🌟", Status = "Café ☕", Country = "🇪🇸 España" },
+                new { Name = "陈小明 🐉", Status = "茶 🍵", Country = "🇨🇳 中国" },
+                new { Name = "Владимир 🚀", Status = "работа 💼", Country = "🇷🇺 Россия" }
+            };
+
+            // Test UTF8 format (default) - this should preserve all emojis and Unicode characters
+            var csvContent = testData.ToCsvString(format: CSVFormat.UTF8);
+            
+            // Verify all emojis are preserved
+            Assert.IsTrue(csvContent.Contains("😀"), "Should contain smile emoji");
+            Assert.IsTrue(csvContent.Contains("🎉"), "Should contain party emoji");
+            Assert.IsTrue(csvContent.Contains("🇺🇸"), "Should contain US flag emoji");
+            Assert.IsTrue(csvContent.Contains("🌟"), "Should contain star emoji");
+            Assert.IsTrue(csvContent.Contains("☕"), "Should contain coffee emoji");
+            Assert.IsTrue(csvContent.Contains("🇪🇸"), "Should contain Spain flag emoji");
+            Assert.IsTrue(csvContent.Contains("🐉"), "Should contain dragon emoji");
+            Assert.IsTrue(csvContent.Contains("🍵"), "Should contain tea emoji");
+            Assert.IsTrue(csvContent.Contains("🇨🇳"), "Should contain China flag emoji");
+            Assert.IsTrue(csvContent.Contains("🚀"), "Should contain rocket emoji");
+            Assert.IsTrue(csvContent.Contains("💼"), "Should contain briefcase emoji");
+            Assert.IsTrue(csvContent.Contains("🇷🇺"), "Should contain Russia flag emoji");
+
+            // Verify special characters from different languages
+            Assert.IsTrue(csvContent.Contains("María"), "Should contain accented characters");
+            Assert.IsTrue(csvContent.Contains("陈小明"), "Should contain Chinese characters");
+            Assert.IsTrue(csvContent.Contains("Владимир"), "Should contain Cyrillic characters");
+            Assert.IsTrue(csvContent.Contains("茶"), "Should contain Chinese tea character");
+            Assert.IsTrue(csvContent.Contains("работа"), "Should contain Russian text");
+
+            // Test file output with BOM for Excel compatibility
+            string emojiTestFile = "emoji_test.csv";
+            testData.WriteCsv(emojiTestFile, format: CSVFormat.UTF8);
+
+            // Verify file was created
+            Assert.IsTrue(System.IO.File.Exists(emojiTestFile), "Emoji CSV file should be created");
+
+            // Read raw bytes to verify BOM is present for Excel compatibility
+            byte[] fileBytes = System.IO.File.ReadAllBytes(emojiTestFile);
+            
+            // Check for UTF-8 BOM (0xEF, 0xBB, 0xBF)
+            Assert.IsTrue(fileBytes.Length >= 3, "File should have at least 3 bytes");
+            Assert.AreEqual(0xEF, fileBytes[0], "First byte should be 0xEF (UTF-8 BOM)");
+            Assert.AreEqual(0xBB, fileBytes[1], "Second byte should be 0xBB (UTF-8 BOM)");
+            Assert.AreEqual(0xBF, fileBytes[2], "Third byte should be 0xBF (UTF-8 BOM)");
+
+            Console.WriteLine("✅ Emoji CSV Test Results:");
+            Console.WriteLine($"📁 File size: {fileBytes.Length} bytes");
+            Console.WriteLine($"📋 UTF-8 BOM detected: {fileBytes[0] == 0xEF && fileBytes[1] == 0xBB && fileBytes[2] == 0xBF}");
+            Console.WriteLine($"🎯 Contains emojis: {csvContent.Contains("😀")}");
+            Console.WriteLine($"🌍 Contains international text: {csvContent.Contains("陈小明")}");
+            
+            Console.WriteLine("\n📝 For Excel users:");
+            Console.WriteLine("1. The generated CSV uses UTF-8 with BOM for maximum compatibility");
+            Console.WriteLine("2. If Excel shows one column, use Data > From Text/CSV and select UTF-8 encoding");
+            Console.WriteLine("3. All emojis and international characters should display correctly");
+
+            // Clean up
+            //if (System.IO.File.Exists(emojiTestFile))
+               // System.IO.File.Delete(emojiTestFile);
+        }
     }
 }

src/DataPowerTools/Csv/CSV.cs

@@ -14,7 +14,7 @@ namespace DataPowerTools
 {
     public static class Csv
     {
-        private static StreamWriter CreateBomAwareStreamWriter(string filePath, CSVFormat format = CSVFormat.ANSI)
+        private static StreamWriter CreateBomAwareStreamWriter(string filePath, CSVFormat format = CSVFormat.UTF8)
         {
             return new StreamWriter(filePath, false, format.GetEncoding());
         }
@@ -88,12 +88,17 @@ public static DataSet GetDataSet(Stream fileStream, char csvDelimiter = ',', boo
 
         /// <summary>
         /// Writes an enumerable of object arrays into a file.
+        /// 
+        /// Excel Compatibility Notes:
+        /// - Default UTF-8 with BOM format ensures emoji and international character support
+        /// - Uses RFC 4180 compliant formatting with CRLF line endings for Windows compatibility
+        /// - If Excel shows one column, use Data > From Text/CSV and select UTF-8 encoding
         /// </summary>
-        /// <param name="rowObjects"></param>
-        /// <param name="headers"></param>
-        /// <param name="outputFile"></param>
-        /// <param name="format">The format to use for the CSV output</param>
-        public static void Write(IEnumerable<object[]> rowObjects, IEnumerable<string> headers, string outputFile, CSVFormat format = CSVFormat.ANSI)
+        /// <param name="rowObjects">The data rows to write</param>
+        /// <param name="headers">Column headers</param>
+        /// <param name="outputFile">Output file path</param>
+        /// <param name="format">The format to use for the CSV output (UTF8 with BOM recommended for Excel)</param>
+        public static void Write(IEnumerable<object[]> rowObjects, IEnumerable<string> headers, string outputFile, CSVFormat format = CSVFormat.UTF8)
         {
             using var sw = CreateBomAwareStreamWriter(outputFile, format);
             using var csvWriter = new CSVWriter(sw, CSVWriter.DefaultSeparator, CSVWriter.DefaultQuoteCharacter, CSVWriter.DefaultEscapeCharacter, CSVWriter.Rfc4180LineEnd);
@@ -109,12 +114,17 @@ public static void Write(IEnumerable<object[]> rowObjects, IEnumerable<string> h
 
         /// <summary>
         /// Writes an IDataReader to a CSV file onto disk (streaming operation).
+        /// 
+        /// Excel Compatibility Notes:
+        /// - Default UTF-8 with BOM format ensures emoji and international character support
+        /// - Uses RFC 4180 compliant formatting with CRLF line endings for Windows compatibility
+        /// - If Excel shows one column, use Data > From Text/CSV and select UTF-8 encoding
         /// </summary>
-        /// <param name="reader"></param>
-        /// <param name="outputFile"></param>
-        /// <param name="writeHeaders"></param>
-        /// <param name="format">The format to use for the CSV output</param>
-        public static void Write(IDataReader reader, string outputFile, bool writeHeaders = true, CSVFormat format = CSVFormat.ANSI)
+        /// <param name="reader">The data reader to write</param>
+        /// <param name="outputFile">Output file path</param>
+        /// <param name="writeHeaders">Whether to include column headers</param>
+        /// <param name="format">The format to use for the CSV output (UTF8 with BOM recommended for Excel)</param>
+        public static void Write(IDataReader reader, string outputFile, bool writeHeaders = true, CSVFormat format = CSVFormat.UTF8)
         {
             using var sw = CreateBomAwareStreamWriter(outputFile, format);
             using var csvWriter = new CSVWriter(sw, CSVWriter.DefaultSeparator, CSVWriter.DefaultQuoteCharacter, CSVWriter.DefaultEscapeCharacter, CSVWriter.Rfc4180LineEnd);
@@ -172,13 +182,18 @@ void Initialize()
 
         /// <summary>
         /// Writes an IDataReader to a CSV string.
+        /// 
+        /// Excel Compatibility Notes:
+        /// - Default UTF-8 with BOM format ensures emoji and international character support
+        /// - Uses RFC 4180 compliant formatting with CRLF line endings for Windows compatibility
+        /// - String output doesn't include BOM - use file output methods for Excel compatibility
         /// </summary>
-        /// <param name="reader"></param>
-        /// <param name="writeHeaders"></param>
-        /// <param name="useTabFormat">Outputs CSV in tab format</param>
-        /// <param name="format">The format to use for the CSV output</param>
-        /// <returns></returns>
-        public static string WriteString(IDataReader reader, bool writeHeaders = true, bool useTabFormat = false, CSVFormat format = CSVFormat.ANSI)
+        /// <param name="reader">The data reader to convert</param>
+        /// <param name="writeHeaders">Whether to include column headers</param>
+        /// <param name="useTabFormat">Outputs CSV in tab format (TSV)</param>
+        /// <param name="format">The format to use for the CSV output (affects character encoding for file writes)</param>
+        /// <returns>CSV formatted string</returns>
+        public static string WriteString(IDataReader reader, bool writeHeaders = true, bool useTabFormat = false, CSVFormat format = CSVFormat.UTF8)
         {
             using var sw = new StringWriter();
 

src/DataPowerTools/Csv/CSVFormat.cs

@@ -1,34 +1,41 @@
+using System;
 using System.Text;
 
 namespace DataPowerTools
 {
     /// <summary>
-    /// Specifies the format for CSV output
+    /// Specifies the format for CSV output with Excel compatibility considerations.
+    /// 
+    /// For optimal Excel compatibility and emoji support, use UTF8 (default).
+    /// If Excel shows one column, use Data > From Text/CSV and select UTF-8 encoding.
     /// </summary>
     public enum CSVFormat
     {
         /// <summary>
-        /// ANSI encoding (Windows-1252) - Default
+        /// ANSI encoding (Windows-1252) - Falls back to UTF-8 if not available.
+        /// Limited emoji support. Use UTF8 for better compatibility.
         /// </summary>
         ANSI,
 
         /// <summary>
-        /// UTF-8 encoding with BOM
+        /// UTF-8 encoding with BOM - Default and recommended for Excel compatibility.
+        /// Supports all emojis and international characters. Excel-friendly with proper import.
         /// </summary>
         UTF8,
 
         /// <summary>
-        /// UTF-16 encoding (Unicode)
+        /// UTF-16 encoding (Unicode) - Full Unicode support but larger file size.
+        /// May require specific Excel import settings.
         /// </summary>
         UTF16,
 
         /// <summary>
-        /// UTF-16 Little Endian encoding
+        /// UTF-16 Little Endian encoding - Full Unicode support but larger file size.
         /// </summary>
         UTF16LE,
 
         /// <summary>
-        /// UTF-16 Big Endian encoding
+        /// UTF-16 Big Endian encoding - Full Unicode support but larger file size.
         /// </summary>
         UTF16BE
     }
@@ -45,13 +52,36 @@ public static Encoding GetEncoding(this CSVFormat format)
         {
             return format switch
             {
-                CSVFormat.ANSI => Encoding.GetEncoding(1252), // Windows-1252
+                CSVFormat.ANSI => GetAnsiEncoding(),
                 CSVFormat.UTF8 => new UTF8Encoding(true), // UTF-8 with BOM
                 CSVFormat.UTF16 => Encoding.Unicode, // UTF-16 LE
                 CSVFormat.UTF16LE => Encoding.Unicode, // UTF-16 LE
                 CSVFormat.UTF16BE => Encoding.BigEndianUnicode, // UTF-16 BE
-                _ => Encoding.GetEncoding(1252) // Default to ANSI
+                _ => new UTF8Encoding(true) // Default to UTF-8 with BOM
             };
         }
+
+        /// <summary>
+        /// Gets ANSI encoding (Windows-1252) with fallback to UTF-8 for .NET Core/.NET 5+
+        /// </summary>
+        private static Encoding GetAnsiEncoding()
+        {
+            try
+            {
+                // Try to get Windows-1252 encoding
+                return Encoding.GetEncoding(1252);
+            }
+            catch (NotSupportedException)
+            {
+                // In .NET Core/.NET 5+, legacy encodings may not be available
+                // Fall back to UTF-8 which supports all the same characters and more
+                return new UTF8Encoding(false); // UTF-8 without BOM for ANSI compatibility
+            }
+            catch (ArgumentException)
+            {
+                // Fallback for any other encoding issues
+                return new UTF8Encoding(false);
+            }
+        }
     }
 } 

src/DataPowerTools/Extensions/DataReaderExtensions.cs

@@ -19,25 +19,33 @@ namespace DataPowerTools.Extensions
     public static class DataReaderExtensions
     {
         /// <summary>
-        /// Writes datareader to CSV.
+        /// Writes datareader to CSV file with Excel-friendly formatting.
+        /// 
+        /// Excel Compatibility:
+        /// - Uses UTF-8 with BOM by default for emoji and international character support
+        /// - RFC 4180 compliant with CRLF line endings
+        /// - If Excel shows one column, use Data > From Text/CSV and select UTF-8 encoding
         /// </summary>
         /// <param name="reader">The data reader to write</param>
         /// <param name="outputFile">The output file path</param>
-        /// <param name="format">The format to use for the CSV output</param>
-        public static void WriteCsv(this IDataReader reader, string outputFile, CSVFormat format = CSVFormat.ANSI)
+        /// <param name="format">The format to use for the CSV output (UTF8 with BOM recommended for Excel)</param>
+        public static void WriteCsv(this IDataReader reader, string outputFile, CSVFormat format = CSVFormat.UTF8)
         {
             Csv.Write(reader, outputFile, format: format);
         }
 
         /// <summary>
-        /// Writes datareader to CSV.
+        /// Converts datareader to CSV string with Excel-friendly formatting.
+        /// 
+        /// Note: String output doesn't include BOM. For Excel compatibility with emojis/international characters,
+        /// use WriteCsv() method to write directly to file which includes proper BOM.
         /// </summary>
-        /// <param name="reader">The data reader to write</param>
+        /// <param name="reader">The data reader to convert</param>
         /// <param name="writeHeaders">Whether to write headers</param>
-        /// <param name="useTabFormat">Whether to use tab format</param>
+        /// <param name="useTabFormat">Whether to use tab format (TSV)</param>
         /// <param name="format">The format to use for the CSV output</param>
         /// <returns>The CSV string</returns>
-        public static string AsCsv(this IDataReader reader, bool writeHeaders = true, bool useTabFormat = false, CSVFormat format = CSVFormat.ANSI)
+        public static string AsCsv(this IDataReader reader, bool writeHeaders = true, bool useTabFormat = false, CSVFormat format = CSVFormat.UTF8)
         {
             return Csv.WriteString(reader, writeHeaders, useTabFormat, format);
         }