高分求救，如何使用httpwebrequest

zyj_soft 2011-02-19 10:55:27

httpwebrequest类，如何取到最新的页面数据，不取缓存中的数据，不使用缓存，每次取实际页面的数据

...全文

224 6 打赏收藏转发到动态举报

写回复

用AI写文章

6 条回复

切换为时间正序

请发表友善的回复…

发表回复

huangwenquan123 2011-02-19

打赏
举报



//显示在页面中的内容是什么抓取的就是什么

            string url = "http://topic.csdn.net/u/20110219/22/4191d9b6-6560-40d5-a29f-280b434acd7a.html";

            string path = Server.MapPath("HttpWebRequest.txt");

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

            HttpWebResponse response = (HttpWebResponse)request.GetResponse();

            if (response.StatusCode == HttpStatusCode.OK)

            {

                using (StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.UTF8))

                {

                    using (StreamWriter writer = new StreamWriter(path, true, Encoding.UTF8))

                    {

                        writer.Write(reader.ReadToEnd());

                    }

                }

            }

zyj_soft 2011-02-19

打赏
举报

我也是这么写的，老读的是缓存在电脑上的网页文件数据

zyj_soft 2011-02-19

打赏
举报

上面写的好像都没控制httpwebrequest取的网页数据是缓存在电脑上的文件

wuyq11 2011-02-19

打赏
举报

// 获取网页的HTML内容，指定Encoding
static string GetHtml(string url, Encoding encoding)
{
byte[] buf = new WebClient().DownloadData(url);
if (encoding != null) return encoding.GetString(buf);
string html = Encoding.UTF8.GetString(buf);
encoding = GetEncoding(html);
if (encoding == null || encoding == Encoding.UTF8) return html;
return encoding.GetString(buf);
}

// 根据网页的HTML内容提取网页的Encoding
static Encoding GetEncoding(string html)
{
string pattern = @"(?i)\bcharset=(?<charset>[-a-zA-Z_0-9]+)";
string charset = Regex.Match(html, pattern).Groups["charset"].Value;
try { return Encoding.GetEncoding(charset); }
catch (ArgumentException) { return null; }
}

wuyq11 2011-02-19

打赏
举报

System.Net.HttpWebRequest request = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";
using(System.Net.WebResponse response = request.GetResponse();
System.IO.Stream resStream = response.GetResponseStream())
{
System.IO.StreamReader sr = new System.IO.StreamReader(resStream, encoding);
string html = (sr.ReadToEnd());
resStream.Close();
sr.Close();
}

System.Net.WebClient wc = new System.Net.WebClient();
wc.Credentials = System.Net.CredentialCache.DefaultCredentials;
Byte[] pageData = wc.DownloadData(PageUrl);
string Content= System.Text.Encoding.Default.GetString(pageData);

子夜__ 2011-02-19

打赏
举报

[code=C]using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.IO.Compression;
using System.Text.RegularExpressions;

namespace WikiPageCreater.Common
{
public class PageHelper
{
/// <summary>
/// 根据 url 获取网页编码
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public static string GetEncoding(string url)
{
HttpWebRequest request = null;
HttpWebResponse response = null;
StreamReader reader = null;
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 20000;
request.AllowAutoRedirect = false;

response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
{
if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress));
else
reader = new StreamReader(response.GetResponseStream(), Encoding.ASCII);

string html = reader.ReadToEnd();

Regex reg_charset = new Regex(@"charset\b\s*=\s*(?<charset>[^""]*)");
if (reg_charset.IsMatch(html))
{
return reg_charset.Match(html).Groups["charset"].Value;
}
else if (response.CharacterSet != string.Empty)
{
return response.CharacterSet;
}
else
return Encoding.Default.BodyName;
}
}
catch
{
}
finally
{

if (response != null)
{
response.Close();
response = null;
}
if (reader != null)
reader.Close();

if (request != null)
request = null;

}

return Encoding.Default.BodyName;
}

/// <summary>
/// 根据 url 和 encoding 获取当前url页面的 html 源代码
/// </summary>
/// <param name="url"></param>
/// <param name="encoding"></param>
/// <returns></returns>
public static string GetHtml(string url, Encoding encoding)
{
HttpWebRequest request = null;
HttpWebResponse response = null;
StreamReader reader = null;
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 20000;
request.AllowAutoRedirect = false;

response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
{
if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress), encoding);
else
reader = new StreamReader(response.GetResponseStream(), encoding);
string html = reader.ReadToEnd();

return html;
}
}
catch
{
}
finally
{

if (response != null)
{
response.Close();
response = null;
}
if (reader != null)
reader.Close();

if (request != null)
request = null;

}

return string.Empty;
}
}
}
[/code]

最新数据是什么意思？

C#使用HttpWebRequest自动更新客户端应用程序

1.这是一个用HttpWebRequest类构建完整Http多部请求上传文件的示例 2.上传地址是金山文档预览的地址，所以，只要上传的是word、pdf、excel一类的文档，可以通过返回的地址直接在线预览，是一种偷懒的文档在线浏览...

C#(也称Csharp)在多线程下并发执行HTTP请求的实现，采用C#封装HttpWebRequest类开发的多线程并发采集程序源码文档，文档中详细说明了HttpWebRequest并发HTTP请求实现网站采集的方法，经过测试同时并发1000+不是问题...

使用Httpwebrequest抓取页面数据主要用于post get数据之类的提交参数

WebApi和访问WebApi两个项目,有源码，注释非常详细，希望对大家有帮助。 HttpWebRequest类访问webApi