Basic web request & simple spider patterns
CookieContainer cc = new CookieContainer();
// dumpreq.aspx displays param, header, and server variable info to help verify request details.
HttpWebResponse req = (HttpWebRequest)WebRequest.Create("http://localhost/learn/utility/dumpreq.aspx");
// Reuse the same cookie container on multiple requests to accumulate cookies returned with responses.
req.CookieContainer = cc;
// pretend to be IE 6, failing to set a UserAgent often leads to a server error 500 response.
req.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705)";
// The following headers are sent by IE 6 but aren’t always necessary…
req.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
req.Headers.Add("Cache-Control:no-cache");
req.Headers.Add("Accept-Encoding:gzip, deflate");
req.Headers.Add("Accept-Language:en-us");
// To use the post method instead of get, add the following…
req.Method = "POST";
req.ContentType = "application/x-www-form-urlencoded";
string formFields = String.Format("logout=false&goto_event=9685&email=foo@bar.com&password=foobar");
StreamWriter sw = new StreamWriter(req.GetRequestStream()); sw.Write(formFields); sw.Close();
// Submit the request and obtain a response…
HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
// To obtain, html content…
string html = (new StreamReader(req.GetResponse().GetResponseStream())).ReadToEnd();
// To obtain binary content…
byte[] data = (new BinaryReader(req.GetResponse().GetResponseStream())).ReadBytes((int)resp.ContentLength);
FileStream bw = File.Create(“image.jpg”); bw.Write(data, 0, (int)resp.ContentLength); bw.Close();