mjzammit 0 Newbie Poster

Hi
I am writing in c# and building a web application to crawl a website
I am getting this error: InvalidOperationException was unhandled by user code
The error is occuring in the highlighted area:

    public static List<string> _goodUrls = new List<string>();
    public static List<string> _pageCrawled = new List<string>();

    public static void CrawlPageforHREF(string url)
    {
        if (!PageHasBeenCrawler(url))
        {
            string htmlText = ClassLibrary1.Find.GetPage(url);
            _pageCrawled.Add(url);

            if (htmlText == null)
            {
                return;
            }

            ParseHTML parse = new ParseHTML();
            parse.Source = htmlText;
            while (!parse.Eof())
            {
                char ch = parse.Parse();
                if (ch == 0)
                {
                    AttributeList tag = parse.GetTag();
                    if (tag["href"] != null)
                    {
                        if ((tag["href"].Value != "") && (tag["href"].Value != "#") && (tag["href"].Value != "/"))
                        {
                            if (IsAWebPage(tag["href"].Value))
                            {
                                //string newValue;

                                if (tag["href"].Value.Substring(0, 1) == "/")

                                {
                                    string newValue = (StartingUrl) + tag["href"].Value;
                                    if (!IsExternalUrl(StartingUrl, newValue))
                                    {
                                        if (!PageHasBeenAdded(newValue))
                                        {
                                            _goodUrls.Add(newValue);
                                        }
                                    }
                                }
                                else if (tag["href"].Value.IndexOf("Default") != -1)
                                {
                                    string newValue = StartingUrl + tag["href"].Value;
                                    if (!IsExternalUrl(StartingUrl, newValue))
                                    {
                                        if (!PageHasBeenAdded(newValue))
                                        {
                                            _goodUrls.Add(newValue);
                                        }
                                    }
                                }
                                else
                                {
                                    string newValue = tag["href"].Value;
                                    if (!IsExternalUrl(StartingUrl, newValue))
                                    {
                                        if (!PageHasBeenAdded(newValue))
                                        {
                                            _goodUrls.Add(newValue);
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }

            foreach (string link [COLOR="Red"][B]in[/B][/COLOR] _goodUrls)
            {
                CrawlHrefs(link);
            }
    }

does anyone have any ideas how to solve???

Be a part of the DaniWeb community

We're a friendly, industry-focused community of developers, IT pros, digital marketers, and technology enthusiasts meeting, networking, learning, and sharing knowledge.