35 lines
604 B
Go
35 lines
604 B
Go
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"log"
|
|
"os"
|
|
|
|
_ "github.com/lib/pq"
|
|
|
|
"git.sr.ht/~sircmpwn/searchhut/config"
|
|
"git.sr.ht/~sircmpwn/searchhut/crawler"
|
|
)
|
|
|
|
func main() {
|
|
domain := os.Args[1]
|
|
|
|
conf := config.Load()
|
|
connstr, ok := conf.Get("searchhut", "connection-string")
|
|
if !ok {
|
|
log.Fatal("Configuration missing connection string")
|
|
}
|
|
|
|
db, err := sql.Open("postgres", connstr)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
ua, ok := conf.Get("searchhut", "user-agent")
|
|
if !ok {
|
|
log.Fatal("Configuration missing user agent")
|
|
}
|
|
|
|
crawler := crawler.NewCrawler(ua, db, domain)
|
|
crawler.Crawl()
|
|
}
|