Merge pull request #10 from olangness/drew

Drew

Author: olangness

WebCrawler/Model/Crawler.cs

@@ -10,8 +10,6 @@
 using WebCrawler.Interfaces;
 using System.Text.RegularExpressions;
 using WebCrawler.Repositories;
-using InfinityCrawler.Processing.Content;
-using TurnerSoftware.RobotsExclusionTools;
 
 namespace WebCrawler.Model
 {
@@ -24,8 +22,9 @@ class Crawler
         private static List<Page> _pages = new List<Page>();
         private static List<string> _exceptions = new List<string>();
         private bool isCurrentPage = true;
-        //public List<String> Disallows = new List<String>();
         private static List<string> urlsWithTopics = new List<string>();
+        private static List<Link> links = new List<Link>();
+        private static List<Log> log = new List<Log>();
 
         //Constructor
         public Crawler(IRepos externalUrlRepository, IRepos otherUrlRepository, IRepos failedUrlRepository, IRepos currentPageUrlRepository)
@@ -70,6 +69,9 @@ private void CrawlPage(string url)
                 page.Url = url;
 
                 _pages.Add(page);
+                links.Add(url);
+                log.Add("New Log Entry: "+url);
+
 
                 //AddUrlToList(topic);
 
@@ -90,12 +92,6 @@ private void CrawlPage(string url)
 
                 isCurrentPage = false;
 
-                /*//For each extracted URL
-                //• Obey robots.txt (freshness caveat)
-                //c. Check that not already in frontier
-                var paths = page.OutLinks.Where(x => page.RobotsAreObeyed(x)
-                                        && b.Contains(x) == false);*/
-
                 //Crawl all the links found on the page.
                 foreach (string link in _externalUrlRepository.List)
                 {
@@ -129,19 +125,6 @@ public static bool PageHasBeenCrawled(string url)
             return false;
         }
 
-        // Adds url containing keyword(s) to a list
-        private static void AddUrlToList(string topic)
-        {
-            foreach (Page page in _pages)
-            {
-                if (page.ToString().Contains(topic))
-                {
-                    urlsWithTopics.Add(page.Url);
-                }
-            }
-        }
-
-
         // Fixes a path. Makes sure it is a fully functional absolute url
         public static string FixPath(string originatingUrl, string link)
         {

WebCrawler/Model/Data Storage/Data.cs

@@ -4,7 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace WindowsFormsApp1
+namespace WebCrawler.Model
 {
     class Data
     {

WebCrawler/Model/Data Storage/Link.cs

@@ -4,7 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace WindowsFormsApp1
+namespace WebCrawler.Model
 {
     class Link
     {

WebCrawler/Model/Data Storage/Log.cs

@@ -4,7 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace WindowsFormsApp1
+namespace WebCrawler.Model
 {
     class Log
     {

WebCrawler/Model/Data Storage/Topic.cs

@@ -4,7 +4,7 @@
 using System.Text;
 using System.Threading.Tasks;
 
-namespace WindowsFormsApp1
+namespace WebCrawler.Model
 {
     class Topic
     {