package main

import "os"
import "sort"
import "golang.org/x/net/html"
import "log"
import "fmt"
import "github.com/PuerkitoBio/goquery"
import "github.com/kennygrant/sanitize"
import "strings"
import "flag"
import "errors"
import "regexp"

type document struct {
	fname string;
	title []string;
	text []string;
	length int;
}

type index struct {
	doc *document;
	title bool;
	freq int;
}

type wordSort struct {
	w string;
	root *wordList;
}

type wordList struct {
	this *index
	next *wordList
}

var r, nonAN *regexp.Regexp;
var stopWords []*regexp.Regexp;


func newDocument() *document {
	return &document{"" , nil, nil, 0};
}

func RemoveNode(r, rn *html.Node) {
	var found bool;
	var n, item *html.Node;
	var nodes map[int]*html.Node;
	var i, j int;

	found = false;
	nodes = make(map[int]*html.Node);

	for n = r.FirstChild; n != nil; n = n.NextSibling {
		if n == rn {
			found = true;
			n.Parent.RemoveChild(n);
		}

		nodes[i] = n;
		i++;
	}

	if !found {
		for j = 0; j < i; j++ {
			item = nodes[j];
			RemoveNode(item, rn);
		}
	}
}
func RemoveTag(doc *goquery.Selection, tag string) {
	doc.Find(tag).Each(func(i int, s *goquery.Selection) {
		RemoveNode(doc.Get(0), s.Get(0));
	});
}

func logReg(h []byte) []byte {
	log.Printf("RegExp: %s", h);
	return h;
}

func parseDoc(fd *os.File, f_info os.FileInfo) (*document, error) {
	var err error;
	var text, t_text string;
	var doc *goquery.Document;
	var body, title *goquery.Selection;
	var r_doc *document;
	var i int;

	doc, err = goquery.NewDocumentFromReader(fd);
	if err != nil {
		log.Printf("goquery error: %s\n", err);
		return nil, errors.New("Can't create goquery documnt");
	}

	body = doc.Find("body");
	RemoveTag(body, "script");
	RemoveTag(body, "noscript");

	title = doc.Find("title");

	//TODO add error detection
	text, err = body.Html();
	t_text, err = title.Html();


	text = r.ReplaceAllString(text, "> <");
	t_text = r.ReplaceAllString(t_text, "> <");

	text = sanitize.HTML(text);
	t_text = sanitize.HTML(t_text);

	text = strings.ToLower(text);
	t_text = strings.ToLower(t_text);

	text = nonAN.ReplaceAllString(text, " ");
	t_text = nonAN.ReplaceAllString(t_text, " ");


	for i = 0; i < len(stopWords); i++ {
		text = stopWords[i].ReplaceAllString(text, " ");
		t_text = stopWords[i].ReplaceAllString(t_text, " ");
	}
	r_doc = newDocument();

	r_doc.fname = f_info.Name();
	r_doc.text = strings.Fields(text);
	r_doc.title = strings.Fields(t_text);
	r_doc.length = len(r_doc.text) + len(r_doc.title);

	return r_doc, nil;
}
func boolToInt(t bool) int {
	if t {
		return 1;
	}
	return 0;
}

func printIndex(words []wordSort, fd *os.File) {
	var i int;
	var cur *wordList;
	var fname string;
	var t int;
	var freq float64;

	for i = 0; i < len(words); i++ {
		fmt.Fprintf(fd, "%s\n", words[i].w);
		for cur = words[i].root; cur != nil; cur = cur.next {
			fname = cur.this.doc.fname;
			t = boolToInt(cur.this.title);
			freq = float64(cur.this.freq) / float64(cur.this.doc.length);
			
			fmt.Fprintf(fd,"\t%s %d %.3f\n", fname, t, freq);
		}
	}
}

func init() {
	var err error;
	log.SetOutput(os.Stderr);
	r, err = regexp.Compile("><");
	if err != nil {
		panic(err);
	}
	nonAN, err = regexp.Compile("[^a-zA-Z0-9]+");
	if err != nil {
		panic(err);
	}
	//TODO add func to read in stop words from a file;
	stopWords = make([]*regexp.Regexp, 26)
	if err != nil {
		panic(err);
	}
	stopWords[0], err = regexp.Compile("\\W+and\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[1], err = regexp.Compile("\\W+a\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[2], err = regexp.Compile("\\W+an\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[3], err = regexp.Compile("\\W+and\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[4], err = regexp.Compile("\\W+are\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[5], err = regexp.Compile("\\W+as\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[6], err = regexp.Compile("\\W+at\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[7], err = regexp.Compile("\\W+be\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[8], err = regexp.Compile("\\W+by\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[9], err = regexp.Compile("\\W+for\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[10], err = regexp.Compile("\\W+from\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[11], err = regexp.Compile("\\W+has\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[12], err = regexp.Compile("\\W+he\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[13], err = regexp.Compile("\\W+in\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[14], err = regexp.Compile("\\W+is\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[15], err = regexp.Compile("\\W+it\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[16], err = regexp.Compile("\\W+its\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[17], err = regexp.Compile("\\W+of\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[18], err = regexp.Compile("\\W+on\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[19], err = regexp.Compile("\\W+that\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[20], err = regexp.Compile("\\W+the\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[21], err = regexp.Compile("\\W+to\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[22], err = regexp.Compile("\\W+was\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[23], err = regexp.Compile("\\W+were\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[24], err = regexp.Compile("\\W+will\\W+");
	if err != nil {
		panic(err);
	}
	stopWords[25], err = regexp.Compile("\\W+with\\W+");
	if err != nil {
		panic(err);
	}
}

func main() {
	//	var words map[string]index
	var p_dir, w, fname string;
	var err error;
	var i, j int;
	var words map[string]*wordList;
	var cur *wordList;
	var tmp *index;
	var sorted []wordSort;

	var files []os.FileInfo;
	var dir, fd *os.File;
	var dir_info, fd_info os.FileInfo;
	var dir_mode os.FileMode;

	var doc *document;

	flag.StringVar(&p_dir, "d", "./pages", "pages directory");

	flag.Parse();

	words = make(map[string]*wordList);

	dir, err = os.Open(p_dir);
	if err != nil {
		log.Printf("Error accessing \"%s\":\t%s\n", p_dir, err);
		os.Exit(1);
	}

	dir_info, err = dir.Stat();
	dir_mode = dir_info.Mode();

	if !dir_mode.IsDir() {
		log.Printf("\"%s\" is not a directory\n", p_dir);
		os.Exit(1);
	}

	files, err = dir.Readdir(0);
	if err != nil {
		log.Printf("Error reading %s\n", p_dir);
		os.Exit(1);
	}

	for i = 0; i < len(files); i++ {
		fd, err = os.Open(fmt.Sprintf("%s/%s", dir_info.Name(), files[i].Name()));
		fd_info, err = fd.Stat();
		if err != nil {
			log.Printf("Error getting info\n");
			os.Exit(1);
		}
		fname = fd_info.Name();

		if err != nil {
			log.Printf("Error reading %s/%s\n", dir_info.Name(), files[i].Name());
		} else {
			fmt.Printf("Indexing %s...\n", fname);
			doc, err = parseDoc(fd, fd_info);
			if err != nil {
				log.Printf("Error parsing %s/%s\n", dir_info.Name(), files[i].Name());
			} else {
				/* Text */
				for j = 0; j < len(doc.text); j++ {
					w = strings.ToLower(doc.text[j]);

					if words[w] == nil{
						tmp = &index{doc: doc, title: false, freq: 0};
						words[w] = &wordList{this: tmp, next: nil};
					}

					for cur = words[w];cur.next != nil && cur.this.doc.fname != fname; cur = cur.next{}

					if cur.this.doc.fname == fname {
						cur.this.freq++
					} else if cur.next == nil {
						tmp = &index{doc: doc, title: false, freq: 1};
						cur.next = &wordList{this: tmp, next: nil};
					} else {
						panic(fmt.Sprintf("%v", cur));
					}
				}
				/* Title */
				for j = 0; j < len(doc.title); j++ {
					w = strings.ToLower(doc.title[j]);

					if words[w] == nil{
						tmp = &index{doc: doc, title: true, freq: 0};
						words[w] = &wordList{this: tmp, next: nil};
					}

					for cur = words[w];cur.next != nil && cur.this.doc.fname != fname; cur = cur.next{}

					if cur.this.doc.fname == fname {
						cur.this.title = true;
						cur.this.freq++;
					} else if cur.next == nil {
						tmp = &index{doc: doc, title: true, freq: 1};
						cur.next = &wordList{this: tmp, next: nil};
					} else {
						panic(fmt.Sprintf("%v", cur));
					}
				}
			}
		}
		fd.Close();
	}
		sorted = make([]wordSort, len(words));
		i = 0;
		for k,v := range words {
			sorted[i].w = k;
			sorted[i].root = v;
			i++;
		}
		
		sort.Slice(sorted, func(i, j int) bool {
			return sorted[i].w < sorted[j].w;
		});
		
		fd,_ = os.Create("index.dat");
		printIndex(sorted, fd);
		fd.Close();
}