一段多编码兼容的 C# 网页读取关键代码

/// <summary>
/// 读取 Web 页面的内容。
/// </summary>
/// <param name="url">Web 页面的 URL。</param>
/// <param name="useProxy">True 则使用程序配置的代理服务器（如果有），False 则不使用。</param>
/// <param name="streamEnc">指定文本编码。当为 null 时，程序将尝试从 Response Headers 以及 Meta Tags（如果 readMetaData 为 true）中找到编码方式，如果找不到，则使用 Encoding.Default。</param>
/// <param name="readMetaData">指定是否读取 Meta Tags 标记以判断文本编码。操作可能会略延长程序时间。</param>
/// <returns>返回 Web 页面的内容。</returns>
public static string GetPageContent(
    string url,
    bool useProxy,
    Encoding streamEnc,
    bool readMetaData)
{
    WebRequest wr = null;
    WebResponse rsp = null;
    WebProxy wp = null;
    Stream st = null;
    StreamReader reader = null;
    MemoryStream ms = null;
    BinaryReader br = null;
    byte[] buffer = null;
    byte[] cnt = null;
    Encoding enc = streamEnc;

    try
    {
        // Create the webrequest
        wr = WebRequest.Create(url);

        // Init The Proxy Server
        if (NetConfig.UseProxy && useProxy)
        {
            wp = new WebProxy(NetConfig.ProxyServer, NetConfig.ProxyPort);
            wp.BypassProxyOnLocal = true;
            wp.UseDefaultCredentials = false;

            if (NetConfig.UseProxyCredential)
                wp.Credentials = new NetworkCredential(NetConfig.ProxyUser, NetConfig.ProxyPass);

            wr.Proxy = wp;
        }

        // Get the response
        rsp = wr.GetResponse();
        st = rsp.GetResponseStream();

        if (enc == null)
        {
            // Copy the stream to a byte array
            br = new BinaryReader(st);
            ms = new MemoryStream();
            while ((buffer = br.ReadBytes(8192)) != null
                && buffer.Length > 0)
            {
                ms.Write(buffer, 0, buffer.Length);
            }
            cnt = ms.ToArray();

            // clear up
            br.Close(); br = null;
            ms.Close(); ms = null;
            st.Close(); st = null;

            // Try to get the encoding
            if (enc == null)
            {
                int j = 0;
                string encStr = "";

                // try read the header
                if (rsp.Headers[HttpResponseHeader.ContentType] != null)
                {
                    j = rsp.Headers[HttpResponseHeader.ContentType].IndexOf("charset=", StringComparison.OrdinalIgnoreCase);
                    if (j >= 0)
                    {
                        encStr = rsp.Headers[HttpResponseHeader.ContentType].Substring(j + "charset=".Length).Trim();
                        if (!string.IsNullOrEmpty(encStr))
                        {
                            try
                            {
                                enc = Encoding.GetEncoding(encStr);
                            }
                            catch { }
                        }
                    }
                }

                // try read the meta data
                if (enc == null && readMetaData)
                {
                    string pgCnt = "";
                    pgCnt = Encoding.Default.GetString(cnt);
                    Regex regex = new Regex("<meta[^<>]+content=\"[^\"]*charset=(?<charset>[^\"]*)\"");
                    Match mc = regex.Match(pgCnt);
                    if (mc != null && mc.Success)
                    {
                        encStr = mc.Groups["charset"].Value;
                        if (encStr.Equals(Encoding.Default.WebName, StringComparison.OrdinalIgnoreCase))
                        {
                            return pgCnt;
                        }

                        if (!string.IsNullOrEmpty(encStr))
                        {
                            try
                            {
                                enc = Encoding.GetEncoding(encStr);
                            }
                            catch { }
                        }
                    }
                }
            }

            if (enc == null)
                enc = Encoding.Default;

            // 正式读取内容
            return enc.GetString(cnt);
        }
        else
        {
            // 如果已经指定了 Encoding
            // 不需要那么多的步骤
            reader = new StreamReader(st, enc);
            return reader.ReadToEnd();
        }
    }
    catch (Exception ex)
    {
        Debugger.LogException(ex);
        return null;
    }
    finally
    {
        if (br != null)
            br.Close();
        if (ms != null)
            ms.Close();
        if (reader != null)
            reader.Close();
        if (st != null)
            st.Close();
        if (rsp != null)
            rsp.Close();
    }
}