feat: add --no-bom flag for ODBC sqlcmd compatibility

By default, -u (unicode output) includes a UTF-16 LE BOM (FF FE) at the
start of output files. ODBC sqlcmd does not write a BOM.

This adds --no-bom flag to omit the BOM when strict ODBC compatibility
is needed. Usage: sqlcmd -u --no-bom -o output.txt

Changes:
- Add NoBOM field to SQLCmdArguments and Sqlcmd structs
- Add --no-bom flag with descriptive help
- Conditionally use unicode.IgnoreBOM when flag is set
- Update README to document the difference and new flag
- Add tests for --no-bom flag

Author: dlevy-msft-sql

README.md

@@ -150,7 +150,7 @@ The following switches have different behavior in this version of `sqlcmd` compa
   - If both `-N` and `-C` are provided, sqlcmd will use their values for encryption negotiation.
   - To provide the value of the host name in the server certificate when using strict encryption, pass the host name with `-F`. Example: `-Ns -F myhost.domain.com`
   - More information about client/server encryption negotiation can be found at <https://docs.microsoft.com/openspecs/windows_protocols/ms-tds/60f56408-0188-4cd5-8b90-25c6f2423868>
-- `-u` The generated Unicode output file will have the UTF16 Little-Endian Byte-order mark (BOM) written to it.
+- `-u` The generated Unicode output file will have the UTF16 Little-Endian Byte-order mark (BOM) written to it. ODBC sqlcmd does not write a BOM; use `--no-bom` with `-u` if you need strict ODBC compatibility.
 - Some behaviors that were kept to maintain compatibility with `OSQL` may be changed, such as alignment of column headers for some data types.
 - All commands must fit on one line, even `EXIT`. Interactive mode will not check for open parentheses or quotes for commands and prompt for successive lines. The ODBC sqlcmd allows the query run by `EXIT(query)` to span multiple lines.
 - `-i` doesn't handle a comma `,` in a file name correctly unless the file name argument is triple quoted. For example:

cmd/sqlcmd/sqlcmd.go

@@ -66,6 +66,7 @@ type SQLCmdArguments struct {
 	ErrorsToStderr              *int
 	Headers                     int
 	UnicodeOutputFile           bool
+	NoBOM                       bool
 	Version                     bool
 	ColumnSeparator             string
 	ScreenWidth                 *int
@@ -171,6 +172,8 @@ func (a *SQLCmdArguments) Validate(c *cobra.Command) (err error) {
 			err = rangeParameterError("-t", fmt.Sprint(a.QueryTimeout), 0, 65534, true)
 		case a.ServerCertificate != "" && !encryptConnectionAllowsTLS(a.EncryptConnection):
 			err = localizer.Errorf("The -J parameter requires encryption to be enabled (-N true, -N mandatory, or -N strict).")
+		case a.NoBOM && !a.UnicodeOutputFile:
+			err = localizer.Errorf("The --no-bom parameter requires -u (Unicode output file).")
 		}
 	}
 	if err != nil {
@@ -457,6 +460,7 @@ func setFlags(rootCmd *cobra.Command, args *SQLCmdArguments) {
 	rootCmd.Flags().IntVarP(&args.Headers, "headers", "h", 0, localizer.Sprintf("Specifies the number of rows to print between the column headings. Use -h-1 to specify that headers not be printed"))
 
 	rootCmd.Flags().BoolVarP(&args.UnicodeOutputFile, "unicode-output-file", "u", false, localizer.Sprintf("Specifies that all output files are encoded with little-endian Unicode"))
+	rootCmd.Flags().BoolVar(&args.NoBOM, "no-bom", false, localizer.Sprintf("Omit the UTF-16 BOM from Unicode output files. Use with -u for ODBC sqlcmd compatibility"))
 	rootCmd.Flags().StringVarP(&args.ColumnSeparator, "column-separator", "s", "", localizer.Sprintf("Specifies the column separator character. Sets the %s variable.", localizer.ColSeparatorVar))
 	rootCmd.Flags().BoolVarP(&args.TrimSpaces, "trim-spaces", "W", false, localizer.Sprintf("Remove trailing spaces from a column"))
 	_ = rootCmd.Flags().BoolP("multi-subnet-failover", "M", false, localizer.Sprintf("Provided for backward compatibility. Sqlcmd always optimizes detection of the active replica of a SQL Failover Cluster"))
@@ -816,6 +820,7 @@ func run(vars *sqlcmd.Variables, args *SQLCmdArguments) (int, error) {
 	s.SetupCloseHandler()
 	defer s.StopCloseHandler()
 	s.UnicodeOutputFile = args.UnicodeOutputFile
+	s.NoBOM = args.NoBOM
 
 	if args.DisableCmd != nil {
 		s.Cmd.DisableSysCommands(args.errorOnBlockedCmd())

cmd/sqlcmd/sqlcmd_test.go

@@ -75,6 +75,9 @@ func TestValidCommandLineToArgsConversion(t *testing.T) {
 		{[]string{"-u", "-A"}, func(args SQLCmdArguments) bool {
 			return args.UnicodeOutputFile && args.DedicatedAdminConnection
 		}},
+		{[]string{"-u", "--no-bom"}, func(args SQLCmdArguments) bool {
+			return args.UnicodeOutputFile && args.NoBOM
+		}},
 		{[]string{"--version"}, func(args SQLCmdArguments) bool {
 			return args.Version
 		}},
@@ -220,6 +223,7 @@ func TestValidateFlags(t *testing.T) {
 		{[]string{"-a", "100"}, "'-a 100': Packet size has to be a number between 512 and 32767."},
 		{[]string{"-h-4"}, "'-h -4': header value must be either -1 or a value between 1 and 2147483647"},
 		{[]string{"-w", "6"}, "'-w 6': value must be greater than 8 and less than 65536."},
+		{[]string{"--no-bom"}, "The --no-bom parameter requires -u (Unicode output file)."},
 	}
 
 	for _, test := range commands {
@@ -298,6 +302,50 @@ func TestUnicodeOutput(t *testing.T) {
 	}
 }
 
+func TestUnicodeOutputNoBOM(t *testing.T) {
+	o, err := os.CreateTemp("", "sqlcmdnobom")
+	assert.NoError(t, err, "os.CreateTemp")
+	defer os.Remove(o.Name())
+	defer o.Close()
+	args = newArguments()
+	args.InputFile = []string{"testdata/selectutf8.txt"}
+	args.OutputFile = o.Name()
+	args.UnicodeOutputFile = true
+	args.NoBOM = true
+	setAzureAuthArgIfNeeded(&args)
+	vars := sqlcmd.InitializeVariables(args.useEnvVars())
+	setVars(vars, &args)
+
+	exitCode, err := run(vars, &args)
+	assert.NoError(t, err, "run")
+	assert.Equal(t, 0, exitCode, "exitCode")
+	fileBytes, err := os.ReadFile(o.Name())
+	if assert.NoError(t, err, "os.ReadFile") {
+		// With --no-bom, the file should NOT start with FF FE (UTF-16 LE BOM)
+		assert.True(t, len(fileBytes) >= 2, "output file should have content")
+		hasBOM := len(fileBytes) >= 2 && fileBytes[0] == 0xFF && fileBytes[1] == 0xFE
+		assert.False(t, hasBOM, "output file should NOT have BOM when --no-bom is used")
+
+		// Verify content is valid UTF-16 LE by decoding and checking for expected text
+		// UTF-16 LE uses 2 bytes per character, so file size should be even
+		assert.Equal(t, 0, len(fileBytes)%2, "UTF-16 LE output should have even number of bytes")
+		// Decode first few bytes as UTF-16 LE and verify it contains recognizable content
+		if len(fileBytes) >= 4 {
+			// Check for ASCII-range characters encoded as UTF-16 LE (low byte first, high byte = 0)
+			// Common characters like digits, letters have high byte = 0 in UTF-16 LE
+			hasValidUtf16Pattern := false
+			for i := 0; i+1 < len(fileBytes); i += 2 {
+				// In UTF-16 LE, ASCII chars have low byte = char, high byte = 0
+				if fileBytes[i+1] == 0 && fileBytes[i] >= 0x20 && fileBytes[i] < 0x7F {
+					hasValidUtf16Pattern = true
+					break
+				}
+			}
+			assert.True(t, hasValidUtf16Pattern, "output should contain valid UTF-16 LE encoded content")
+		}
+	}
+}
+
 func TestUnicodeInput(t *testing.T) {
 	testfiles := []string{
 		filepath.Join(`testdata`, `selectutf8.txt`),

pkg/sqlcmd/commands.go

@@ -321,9 +321,12 @@ func outCommand(s *Sqlcmd, args []string, line uint) error {
 			return InvalidFileError(err, args[0])
 		}
 		if s.UnicodeOutputFile {
-			// ODBC sqlcmd doesn't write a BOM but we will.
-			// Maybe the endian-ness should be configurable.
-			win16le := unicode.UTF16(unicode.LittleEndian, unicode.UseBOM)
+			// By default we write a BOM, but --no-bom omits it for ODBC sqlcmd compatibility
+			bomPolicy := unicode.UseBOM
+			if s.NoBOM {
+				bomPolicy = unicode.IgnoreBOM
+			}
+			win16le := unicode.UTF16(unicode.LittleEndian, bomPolicy)
 			encoder := transform.NewWriter(o, win16le.NewEncoder())
 			s.SetOutput(encoder)
 		} else {

pkg/sqlcmd/sqlcmd.go

@@ -84,6 +84,8 @@ type Sqlcmd struct {
 	PrintError func(msg string, severity uint8) bool
 	// UnicodeOutputFile is true when UTF16 file output is needed
 	UnicodeOutputFile bool
+	// NoBOM omits the BOM from UTF-16 output files (ODBC sqlcmd compatibility)
+	NoBOM bool
 	// EchoInput tells the GO command to print the batch text before running the query
 	EchoInput bool
 	colorizer color.Colorizer