UTF-8 text file with a BOM

Handle it in code

Each snippet below was run against this exact file with Python 3.14.4 and Node 24.15.0 when this page was built, and the output shown is what it printed.

Python

Let a byte order mark decide the encoding when there is one, and otherwise insist on UTF-8. Guessing silently is how text turns into mojibake.

import codecs

BOMS = ((codecs.BOM_UTF8, "utf-8"), (codecs.BOM_UTF16_LE, "utf-16-le"), (codecs.BOM_UTF16_BE, "utf-16-be"))

def decode(raw):
    for bom, enc in BOMS:
        if raw.startswith(bom):
            return enc + " with BOM", raw[len(bom):].decode(enc)
    return "utf-8", raw.decode("utf-8")   # raises on anything else: fail loudly

with open("utf8-with-bom.txt", "rb") as f:
    raw = f.read()
try:
    enc, text = decode(raw)
    print(enc, repr(text[:40]))
except UnicodeDecodeError as e:
    print(f"not UTF-8: {e.reason} at byte {e.start}")

Run against this file, it prints: utf-8 with BOM 'The quick brown fox jumps over the lazy '

Node.js

TextDecoder with fatal: true throws on invalid bytes instead of inserting replacement characters.

const fs = require('node:fs');

const raw = fs.readFileSync('utf8-with-bom.txt');
let enc = 'utf-8', start = 0;
if (raw[0] === 0xef && raw[1] === 0xbb && raw[2] === 0xbf) start = 3;
else if (raw[0] === 0xff && raw[1] === 0xfe) { enc = 'utf-16le'; start = 2; }
else if (raw[0] === 0xfe && raw[1] === 0xff) { enc = 'utf-16be'; start = 2; }

try {
  const text = new TextDecoder(enc, { fatal: true, ignoreBOM: true }).decode(raw.subarray(start));
  console.log(enc + (start ? ' with BOM' : ''), JSON.stringify(text.slice(0, 40)));
} catch (e) {
  console.log('not valid ' + enc + ':', e.message);
}

Run against this file, it prints: utf-8 with BOM "The quick brown fox jumps over the lazy "