ByteExtenstion.cs 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320
  1. using System.Globalization;
  2. using System.Text;
  3. using System.Text.RegularExpressions;
  4. namespace PlcSiemens.Core.Extension
  5. {
  6. public static class ByteExtenstion
  7. {
  8. /// <summary>检测字节数组编码</summary>
  9. public static Encoding Detect(this byte[] data)
  10. {
  11. // 探测BOM头
  12. var encoding = DetectBOM(data);
  13. if (encoding != null) return encoding;
  14. return DetectInternal(data);
  15. }
  16. public static Encoding DetectInternal(this byte[] data)
  17. {
  18. Encoding encoding = null;
  19. // 最笨的办法尝试
  20. var encs = new[] {
  21. // 常用
  22. Encoding.UTF8,
  23. // 用户界面选择语言编码
  24. Encoding.GetEncoding(CultureInfo.CurrentUICulture.TextInfo.ANSICodePage),
  25. // 本地默认编码
  26. Encoding.Default
  27. };
  28. encs = encs.Where(s => s != null).GroupBy(s => s.CodePage).Select(s => s.First()).ToArray();
  29. // 如果有单字节编码,优先第一个非单字节的编码
  30. foreach (var enc in encs)
  31. {
  32. if (IsMatch(data, enc))
  33. {
  34. if (!enc.IsSingleByte) return enc;
  35. if (encoding == null) encoding = enc;
  36. }
  37. }
  38. if (encoding != null) return encoding;
  39. // 探测Unicode编码
  40. encoding = DetectUnicode(data);
  41. if (encoding != null) return encoding;
  42. // 简单方法探测ASCII
  43. encoding = DetectASCII(data);
  44. return encoding;
  45. }
  46. /// <summary>检测BOM字节序</summary>
  47. /// <param name="boms"></param>
  48. /// <returns></returns>
  49. // ReSharper disable once InconsistentNaming
  50. public static Encoding DetectBOM(this byte[] boms)
  51. {
  52. if (boms.Length < 2) return null;
  53. if (boms[0] == 0xff && boms[1] == 0xfe && (boms.Length < 4 || boms[2] != 0 || boms[3] != 0)) return Encoding.Unicode;
  54. if (boms[0] == 0xfe && boms[1] == 0xff) return Encoding.BigEndianUnicode;
  55. if (boms.Length < 3) return null;
  56. if (boms[0] == 0xef && boms[1] == 0xbb && boms[2] == 0xbf) return Encoding.UTF8;
  57. if (boms[0] == 0x2b && boms[1] == 0x2f && boms[2] == 0x76) return Encoding.UTF7;
  58. if (boms.Length < 4) return null;
  59. if (boms[0] == 0xff && boms[1] == 0xfe && boms[2] == 0 && boms[3] == 0) return Encoding.UTF32;
  60. if (boms[0] == 0 && boms[1] == 0 && boms[2] == 0xfe && boms[3] == 0xff) return Encoding.GetEncoding(12001);
  61. return null;
  62. }
  63. /// <summary>检测是否ASCII</summary>
  64. // ReSharper disable once InconsistentNaming
  65. private static Encoding DetectASCII(byte[] data)
  66. {
  67. // 如果所有字节都小于128,则可以使用ASCII编码
  68. if (data.Any(t => t >= 128))
  69. {
  70. return null;
  71. }
  72. return Encoding.ASCII;
  73. }
  74. private static bool IsMatch(byte[] data, Encoding encoding)
  75. {
  76. if (encoding == null) encoding = Encoding.Default;
  77. try
  78. {
  79. var str = encoding.GetString(data);
  80. var buf = encoding.GetBytes(str);
  81. // 考虑到噪声干扰,只要0.9
  82. var score = buf.Length * 9 / 10;
  83. var match = 0;
  84. for (var i = 0; i < buf.Length; i++)
  85. {
  86. if (data[i] == buf[i])
  87. {
  88. match++;
  89. if (match >= score) return true;
  90. }
  91. }
  92. }
  93. catch (Exception ex)
  94. {
  95. // XTrace.WriteException(ex);
  96. // ignored
  97. }
  98. return false;
  99. }
  100. /// <summary>启发式探测Unicode编码</summary>
  101. private static Encoding DetectUnicode(byte[] data)
  102. {
  103. var oddBinaryNullsInSample = 0;
  104. var evenBinaryNullsInSample = 0;
  105. var suspiciousUtf8SequenceCount = 0;
  106. var suspiciousUtf8BytesTotal = 0;
  107. // ReSharper disable once InconsistentNaming
  108. var likelyUSASCIIBytesInSample = 0;
  109. // Cycle through, keeping count of binary null positions, possible UTF-8
  110. // sequences from upper ranges of Windows-1252, and probable US-ASCII
  111. // character counts.
  112. long pos = 0;
  113. // ReSharper disable once InconsistentNaming
  114. var skipUTF8Bytes = 0;
  115. while (pos < data.Length)
  116. {
  117. // 二进制空分布
  118. if (data[pos] == 0)
  119. {
  120. if (pos % 2 == 0)
  121. evenBinaryNullsInSample++;
  122. else
  123. oddBinaryNullsInSample++;
  124. }
  125. // 可见 ASCII 字符
  126. if (IsCommonASCII(data[pos]))
  127. likelyUSASCIIBytesInSample++;
  128. // 类似UTF-8的可疑序列
  129. if (skipUTF8Bytes == 0)
  130. {
  131. int len = DetectSuspiciousUTF8SequenceLength(data, pos);
  132. if (len > 0)
  133. {
  134. suspiciousUtf8SequenceCount++;
  135. suspiciousUtf8BytesTotal += len;
  136. skipUTF8Bytes = len - 1;
  137. }
  138. }
  139. else
  140. {
  141. skipUTF8Bytes--;
  142. }
  143. pos++;
  144. }
  145. // UTF-16
  146. // LE 小端 在英语或欧洲环境,经常使用奇数个0(以0开始),而很少用偶数个0
  147. // BE 大端 在英语或欧洲环境,经常使用偶数个0(以0开始),而很少用奇数个0
  148. if (((evenBinaryNullsInSample * 2.0) / data.Length) < 0.2
  149. && ((oddBinaryNullsInSample * 2.0) / data.Length) > 0.6
  150. )
  151. return Encoding.Unicode;
  152. if (((oddBinaryNullsInSample * 2.0) / data.Length) < 0.2
  153. && ((evenBinaryNullsInSample * 2.0) / data.Length) > 0.6
  154. )
  155. return Encoding.BigEndianUnicode;
  156. // UTF-8
  157. // 使用正则检测,参考http://www.w3.org/International/questions/qa-forms-utf-8
  158. string potentiallyMangledString = Encoding.ASCII.GetString(data);
  159. var reg = new Regex(@"\A("
  160. + @"[\x09\x0A\x0D\x20-\x7E]" // ASCII
  161. + @"|[\xC2-\xDF][\x80-\xBF]" // 不太长的2字节
  162. + @"|\xE0[\xA0-\xBF][\x80-\xBF]" // 排除太长
  163. + @"|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}" // 连续的3字节
  164. + @"|\xED[\x80-\x9F][\x80-\xBF]" // 排除代理
  165. + @"|\xF0[\x90-\xBF][\x80-\xBF]{2}" // 1~3
  166. + @"|[\xF1-\xF3][\x80-\xBF]{3}" // 4~15
  167. + @"|\xF4[\x80-\x8F][\x80-\xBF]{2}" // 16
  168. + @")*\z");
  169. if (reg.IsMatch(potentiallyMangledString))
  170. {
  171. //Unfortunately, just the fact that it CAN be UTF-8 doesn't tell you much about probabilities.
  172. //If all the characters are in the 0-127 range, no harm done, most western charsets are same as UTF-8 in these ranges.
  173. //If some of the characters were in the upper range (western accented characters), however, they would likely be mangled to 2-byte by the UTF-8 encoding process.
  174. // So, we need to play stats.
  175. // The "Random" likelihood of any pair of randomly generated characters being one
  176. // of these "suspicious" character sequences is:
  177. // 128 / (256 * 256) = 0.2%.
  178. //
  179. // In western text data, that is SIGNIFICANTLY reduced - most text data stays in the <127
  180. // character range, so we assume that more than 1 in 500,000 of these character
  181. // sequences indicates UTF-8. The number 500,000 is completely arbitrary - so sue me.
  182. //
  183. // We can only assume these character sequences will be rare if we ALSO assume that this
  184. // IS in fact western text - in which case the bulk of the UTF-8 encoded data (that is
  185. // not already suspicious sequences) should be plain US-ASCII bytes. This, I
  186. // arbitrarily decided, should be 80% (a random distribution, eg binary data, would yield
  187. // approx 40%, so the chances of hitting this threshold by accident in random data are
  188. // VERY low).
  189. // 很不幸运,事实上,它仅仅可能是UTF-8。如果所有字符都在0~127范围,那是没有问题的,绝大部分西方字符在UTF-8都在这个范围。
  190. // 然而如果部分字符在大写区域(西方口语字符),用UTF-8编码处理可能造成误伤。所以我们需要继续分析。
  191. // 随机生成字符成为可疑序列的可能性是:128 / (256 * 256) = 0.2%
  192. // 在西方文本数据,这要小得多,绝大部分文本数据停留在小于127的范围。所以我们假定在500000个字符中多余一个UTF-8字符
  193. if ((suspiciousUtf8SequenceCount * 500000.0 / data.Length >= 1) // 可疑序列
  194. && (
  195. // 所有可疑情况,无法平率ASCII可能性
  196. data.Length - suspiciousUtf8BytesTotal == 0
  197. ||
  198. likelyUSASCIIBytesInSample * 1.0 / (data.Length - suspiciousUtf8BytesTotal) >= 0.8
  199. )
  200. )
  201. return Encoding.UTF8;
  202. }
  203. return null;
  204. }
  205. /// <summary>是否可见ASCII</summary>
  206. /// <param name="bt"></param>
  207. /// <returns></returns>
  208. // ReSharper disable once InconsistentNaming
  209. private static bool IsCommonASCII(byte bt)
  210. {
  211. if (bt == 0x0A // 回车
  212. || bt == 0x0D // 换行
  213. || bt == 0x09 // 制表符
  214. || (bt >= 0x20 && bt <= 0x2F) // 符号
  215. || (bt >= 0x30 && bt <= 0x39) // 数字
  216. || (bt >= 0x3A && bt <= 0x40) // 符号
  217. || (bt >= 0x41 && bt <= 0x5A) // 大写字母
  218. || (bt >= 0x5B && bt <= 0x60) // 符号
  219. || (bt >= 0x61 && bt <= 0x7A) // 小写字母
  220. || (bt >= 0x7B && bt <= 0x7E) // 符号
  221. )
  222. return true;
  223. else
  224. return false;
  225. }
  226. /// <summary>检测可能的UTF8序列长度</summary>
  227. /// <param name="buf"></param>
  228. /// <param name="pos"></param>
  229. /// <returns></returns>
  230. // ReSharper disable once InconsistentNaming
  231. private static int DetectSuspiciousUTF8SequenceLength(byte[] buf, Int64 pos)
  232. {
  233. if (buf.Length > pos + 1)
  234. {
  235. var first = buf[pos];
  236. var second = buf[pos + 1];
  237. if (first == 0xC2)
  238. {
  239. if (second == 0x81 || second == 0x8D || second == 0x8F || second == 0x90 || second == 0x9D || second >= 0xA0 && second <= 0xBF)
  240. return 2;
  241. }
  242. else if (first == 0xC3)
  243. {
  244. if (second >= 0x80 && second <= 0xBF) return 2;
  245. }
  246. else if (first == 0xC5)
  247. {
  248. if (second == 0x92 || second == 0x93 || second == 0xA0 || second == 0xA1 || second == 0xB8 || second == 0xBD || second == 0xBE)
  249. return 2;
  250. }
  251. else if (first == 0xC6)
  252. {
  253. if (second == 0x92) return 2;
  254. }
  255. else if (first == 0xCB)
  256. {
  257. if (second == 0x86 || second == 0x9C) return 2;
  258. }
  259. else if (buf.Length >= pos + 2 && first == 0xE2)
  260. {
  261. var three = buf[pos + 2];
  262. if (second == 0x80)
  263. {
  264. if (three == 0x93 || three == 0x94 || three == 0x98 || three == 0x99 || three == 0x9A)
  265. return 3;
  266. if (three == 0x9C || three == 0x9D || three == 0x9E)
  267. return 3;
  268. if (three == 0xA0 || three == 0xA1 || three == 0xA2)
  269. return 3;
  270. if (three == 0xA6 || three == 0xB0 || three == 0xB9 || three == 0xBA)
  271. return 3;
  272. }
  273. else if (second == 0x82 && three == 0xAC || second == 0x84 && three == 0xA2)
  274. return 3;
  275. }
  276. }
  277. return 0;
  278. }
  279. }
  280. }