Hi
I am writing in c# and building a web application to crawl a website
I am getting this error: InvalidOperationException was unhandled by user code
The error is occuring in the highlighted area:
public static List<string> _goodUrls = new List<string>();
public static List<string> _pageCrawled = new List<string>();
public static void CrawlPageforHREF(string url)
{
if (!PageHasBeenCrawler(url))
{
string htmlText = ClassLibrary1.Find.GetPage(url);
_pageCrawled.Add(url);
if (htmlText == null)
{
return;
}
ParseHTML parse = new ParseHTML();
parse.Source = htmlText;
while (!parse.Eof())
{
char ch = parse.Parse();
if (ch == 0)
{
AttributeList tag = parse.GetTag();
if (tag["href"] != null)
{
if ((tag["href"].Value != "") && (tag["href"].Value != "#") && (tag["href"].Value != "/"))
{
if (IsAWebPage(tag["href"].Value))
{
//string newValue;
if (tag["href"].Value.Substring(0, 1) == "/")
{
string newValue = (StartingUrl) + tag["href"].Value;
if (!IsExternalUrl(StartingUrl, newValue))
{
if (!PageHasBeenAdded(newValue))
{
_goodUrls.Add(newValue);
}
}
}
else if (tag["href"].Value.IndexOf("Default") != -1)
{
string newValue = StartingUrl + tag["href"].Value;
if (!IsExternalUrl(StartingUrl, newValue))
{
if (!PageHasBeenAdded(newValue))
{
_goodUrls.Add(newValue);
}
}
}
else
{
string newValue = tag["href"].Value;
if (!IsExternalUrl(StartingUrl, newValue))
{
if (!PageHasBeenAdded(newValue))
{
_goodUrls.Add(newValue);
}
}
}
}
}
}
}
}
foreach (string link [COLOR="Red"][B]in[/B][/COLOR] _goodUrls)
{
CrawlHrefs(link);
}
}
does anyone have any ideas how to solve???