Hi
I can loginto a site like http://www.abc.com/profile/ with WebRequest.Create and network credentials and capture html code.
But i cannot loginto for http://www.abc.com/some.php?id=123&name=abc with WebRequest.Create and network credentials to capture html code.
How can I do it. My problem is with query string .. ?id=123&name=abc
Jump to PostWhat error do you get when you add the query string - are you sure the web page isnt blocking it due to its manner of coding?
Jump to PostWell. One thing you should do is debug how the web login is handled.
A really good way to do this is to write an elementary proxy which records the URLs you retrieve, the data you send, the data you receieve..
Im going to guess your app fails …
What error do you get when you add the query string - are you sure the web page isnt blocking it due to its manner of coding?
Thank you for reply. Actually I have not found any error. Here I am explaining...
I need to extract data from a website like
http://www.abc.com/index.php?fuseaction=default.user_details&id=22067
In browser, after signing in when I paste the above url the page comes with user details. But before my signing in when paste the above url, a signing in page comes like
http://www.abc.com/index.php?fuseaction=members.login&cookie_check=1
But after signing in
http://www.abc.com/index.php?fuseaction=default.home comes
But my desired page does not come.I have used the following code:
using System;
using System.Collections.Specialized;
using System.Net;
using System.Text;
using System.IO;
using System.Windows.Forms;
namespace BaseClassNameSpace.Web.BaseServices
{
/// <summary>
///This base class provides implementation of request
///and response methods during Http Calls.
/// </summary>
public class HttpBaseClass
{
private string UserName;
private string UserPwd;
private string ProxyServer;
private int ProxyPort;
private string Request;
public HttpBaseClass(string HttpUserName,
string HttpUserPwd, string HttpProxyServer,
int HttpProxyPort, string HttpRequest)
{
UserName = HttpUserName;
UserPwd = HttpUserPwd;
ProxyServer = HttpProxyServer;
ProxyPort = HttpProxyPort;
Request = HttpRequest;
}
/// <summary>
/// This method creates secure/non secure web
/// request based on the parameters passed.
/// </summary>
/// <param name="uri"></param>
/// <param name="collHeader">This parameter of type
/// NameValueCollection may contain any extra header
/// elements to be included in this request </param>
/// <param name="RequestMethod">Value can POST OR GET</param>
/// <param name="NwCred">In case of secure request this would be true</param>
/// <returns></returns>
public virtual HttpWebRequest CreateWebRequest(string uri,
NameValueCollection collHeader,
string RequestMethod, bool NwCred)
{
HttpWebRequest webrequest =
(HttpWebRequest)WebRequest.Create(uri);
webrequest.KeepAlive = false;
webrequest.Method = RequestMethod;
//webrequest.UserAgent = "IE";
//webrequest.RequestUri;
int iCount = collHeader.Count;
string key;
string keyvalue;
for (int i = 0; i < iCount; i++)
{
key = collHeader.Keys[i];
keyvalue = collHeader[i];
webrequest.Headers.Add(key, keyvalue);
}
webrequest.ContentType = "text/html";
//"application/x-www-form-urlencoded";
if (ProxyServer.Length > 0)
{
webrequest.Proxy = new
WebProxy(ProxyServer, ProxyPort);
}
webrequest.AllowAutoRedirect = true;
if (NwCred)
{
CredentialCache wrCache =
new CredentialCache();
wrCache.Add(new Uri(uri), "Basic",
new NetworkCredential(UserName, UserPwd));
webrequest.Credentials = wrCache;
}
//Remove collection elements
collHeader.Clear();
return webrequest;
}//End of secure CreateWebRequest
/// <summary>
/// This method retreives redirected URL from
/// response header and also passes back
/// any cookie (if there is any)
/// </summary>
/// <param name="webresponse"></param>
/// <param name="Cookie"></param>
/// <returns></returns>
public virtual string GetRedirectURL(HttpWebResponse
webresponse, ref string Cookie)
{
string uri = "";
WebHeaderCollection headers = webresponse.Headers;
if ((webresponse.StatusCode == HttpStatusCode.Found) ||
(webresponse.StatusCode == HttpStatusCode.Redirect) ||
(webresponse.StatusCode == HttpStatusCode.Moved) ||
(webresponse.StatusCode == HttpStatusCode.MovedPermanently))
{
// Get redirected uri
uri = headers["Location"];
uri = uri.Trim();
}
//Check for any cookies
if (headers["Set-Cookie"] != null)
{
Cookie = headers["Set-Cookie"];
MessageBox.Show("Cookie" + Cookie.ToString());
MessageBox.Show("Header" + headers.ToString());
}
else MessageBox.Show("Header" + headers["Set-Cookie"]);
// string StartURI = "http:/";
// if (uri.Length > 0 && uri.StartsWith(StartURI)==false)
// {
// uri = StartURI + uri;
// }
return uri;
}//End of GetRedirectURL method
public virtual string GetFinalResponse(string ReUri,
string Cookie, string RequestMethod, bool NwCred)
{
NameValueCollection collHeader =
new NameValueCollection();
if (Cookie.Length > 0)
{
collHeader.Add("Cookie", Cookie);
}
HttpWebRequest webrequest =
CreateWebRequest(ReUri, collHeader,
RequestMethod, NwCred);
BuildReqStream(ref webrequest);
HttpWebResponse webresponse;
webresponse = (HttpWebResponse)webrequest.GetResponse();
Encoding enc = System.Text.Encoding.Default;//.GetEncoding.(1252);
StreamReader loResponseStream = new
StreamReader(webresponse.GetResponseStream(), enc);
string Response = loResponseStream.ReadToEnd();
loResponseStream.Close();
webresponse.Close();
return Response;
}
private void BuildReqStream(ref HttpWebRequest webrequest)
//This method build the request stream for WebRequest
{
byte[] bytes = Encoding.ASCII.GetBytes(Request);
webrequest.ContentLength = bytes.Length;
Stream oStreamOut = webrequest.GetRequestStream();
oStreamOut.Write(bytes, 0, bytes.Length);
oStreamOut.Close();
}
}
}//End of HttpBaseClass class
//“End of Base Class”
using System;
using System.Collections.Specialized;
using System.Net;
using System.Text;
using System.IO;
using BaseClassNameSpace.Web.BaseServices;
using System.Windows.Forms;
namespace DeriveClassNameSpace.Services.Web
{
public class HttpRequestResponse
{
private string URI;
private string Request;
private string UserName;
private string UserPwd;
private string ProxyServer;
private int ProxyPort;
private string RequestMethod = "GET";
public HttpRequestResponse(string pRequest,
string pURI)//Constructor
{
Request = pRequest;
URI = pURI;
}
public string HTTP_USER_NAME
{
get
{
return UserName;
}
set
{
UserName = value;
}
}
public string HTTP_USER_PASSWORD
{
get
{
return UserPwd;
}
set
{
UserPwd = value;
}
}
public string PROXY_SERVER
{
get
{
return ProxyServer;
}
set
{
ProxyServer = value;
}
}
public int PROXY_PORT
{
get
{
return ProxyPort;
}
set
{
ProxyPort = value;
}
}
public string SendRequest()
/*This public interface receives the request
and send the response of type string. */
{
string FinalResponse = "";
string Cookie = "";
NameValueCollection collHeader = new NameValueCollection();
HttpWebResponse webresponse;
HttpBaseClass BaseHttp = new
HttpBaseClass(UserName, UserPwd,
ProxyServer, ProxyPort, Request);
try
{
HttpWebRequest webrequest =
BaseHttp.CreateWebRequest(URI,
collHeader, RequestMethod, true);
webresponse =
(HttpWebResponse)webrequest.GetResponse();
string ReUri =
BaseHttp.GetRedirectURL(webresponse,
ref Cookie);
//Check if there is any redirected URI.
webresponse.Close();
ReUri = ReUri.Trim();
if (ReUri.Length == 0) //No redirection URI
{
ReUri = URI;
}
RequestMethod = "POST";
FinalResponse = BaseHttp.GetFinalResponse(ReUri,
Cookie, RequestMethod, true);
}//End of Try Block
catch (WebException e)
{
throw CatchHttpExceptions(FinalResponse = e.Message);
}
catch (System.Exception e)
{
throw new Exception(FinalResponse = e.Message);
}
finally
{
BaseHttp = null;
}
return FinalResponse;
} //End of SendRequestTo method
private WebException CatchHttpExceptions(string ErrMsg)
{
ErrMsg = "Error During Web Interface. Error is: " + ErrMsg;
return new WebException(ErrMsg);
}
}//End of RequestResponse Class
}
private void button1_Click(object sender, EventArgs e)
{
HttpRequestResponse myReqRes = new HttpRequestResponse("", "http://www.abc.com/index.php?fuseaction=default.user_details&id=22067");
myReqRes.HTTP_USER_NAME = "XXXXX";
myReqRes.HTTP_USER_PASSWORD = "XXXXXX";
myReqRes.PROXY_PORT = 8080;
myReqRes.PROXY_SERVER = "MyProxyServer";
richTextBox1.Text = myReqRes.SendRequest();
}
Well. One thing you should do is debug how the web login is handled.
A really good way to do this is to write an elementary proxy which records the URLs you retrieve, the data you send, the data you receieve..
Im going to guess your app fails due to cookies and redirection.
And before you ask, to make the proxy you need to understand the http protocol and to accept tcp requests on a port, and then on receiving them, connect to the relevant URL and forward all that request on, and then when it responds, send all that data back.. Its a useful app to write and learn.
Yes you are right . I am new at HttpRequest. Can you briefly explain about elementary proxy server? I will debug my code asap. Thanx
Um , I just did!
Can you find anything that can help me?
Well, what have *you* found for yourself so far?
We're a friendly, industry-focused community of developers, IT pros, digital marketers, and technology enthusiasts meeting, networking, learning, and sharing knowledge.