aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--webCrawler2/crawler.go65
1 files changed, 32 insertions, 33 deletions
diff --git a/webCrawler2/crawler.go b/webCrawler2/crawler.go
index 8725df8..1c7bc70 100644
--- a/webCrawler2/crawler.go
+++ b/webCrawler2/crawler.go
@@ -1,16 +1,16 @@
package main
import "crypto/md5"
-import "sync/atomic"
-import "os"
import "fmt"
+import "github.com/PuerkitoBio/goquery"
import "log"
-import "time"
-import "sync"
+import "net/url"
+import "os"
import "strconv"
import "strings"
-import "net/url"
-import "github.com/PuerkitoBio/goquery"
+import "sync"
+import "sync/atomic"
+import "time"
type link struct {
u *url.URL
@@ -19,7 +19,6 @@ type link struct {
var mutex *sync.Mutex
var Prev map[string]bool
-var tFile *os.File
var base string
var links_visited uint64 = 0
@@ -63,7 +62,6 @@ func worker(done chan bool, jobs chan link, depth int, id int, total uint64) {
for {
x := atomic.LoadUint64(&links_visited)
if x >= total {
- fmt.Printf("Max links reached\n\tlinks: %d total: %d\n", x, total)
done <- true
return
}
@@ -79,7 +77,7 @@ func worker(done chan bool, jobs chan link, depth int, id int, total uint64) {
break
}
- fmt.Printf("Adding links from %s depth: %d...\n", j.u.String(), j.depth)
+ fmt.Printf("worker %d Working on %s...\n", id, j.u.String())
consume(doc, j, id)
addLinks(doc, jobs, j, j.depth, id)
@@ -95,18 +93,26 @@ func init() {
mutex = &sync.Mutex{}
Prev = make(map[string]bool)
var err error
- tFile, err = os.Create("./test.txt")
+
+ fi, err := os.Lstat("./pages");
if err != nil {
- panic(err)
+ fmt.Printf("INIT ERROR: %s\n", err);
+ }
+
+ if (fi == nil) {
+ os.Mkdir("./pages", 0755);
+ } else if (fi.Mode().IsRegular()) {
+ panic("pages is not a valid directory\n")
}
+
}
func main() {
var d, w, b int
var t uint64
- if len(os.Args) < 4 {
- fmt.Printf("usage: crawler url [depth max_links] [workers]\n")
+ if len(os.Args) < 5 {
+ fmt.Printf("usage: crawler url depth max_links workers\n")
panic("test")
}
@@ -119,18 +125,9 @@ func main() {
d, _ = strconv.Atoi(os.Args[2])
b, _ = (strconv.Atoi(os.Args[3]))
t = uint64(b)
- if len(os.Args) >= 4 {
- b, _ = (strconv.Atoi(os.Args[3]))
- t = uint64(b)
-
- } else {
- t = 10
- }
- if len(os.Args) == 5 {
- w, _ = strconv.Atoi(os.Args[4])
- } else {
- w = 4
- }
+ b, _ = (strconv.Atoi(os.Args[3]))
+ t = uint64(b)
+ w, _ = strconv.Atoi(os.Args[4])
links := make(chan link, 1024*1024)
docs := make(chan *goquery.Document, 100)
@@ -152,15 +149,17 @@ func main() {
go worker(done, jobs, d, i, t)
}
- for i := 0; i < w*2; {
- <-done
-
- fmt.Printf("%d done\n", i)
- i++
+ for i := 0; i < w; {
+ select {
+ case <-done:
+ i++
+ case <-time.After(1 * time.Second):
+ if len(jobs) == (1024 * 1024) {
+ i = w
+ }
+ }
}
- tFile.Close()
close(done)
- close(links)
- close(docs)
+ close(jobs)
}