init

README.md

@@ -6,5 +6,8 @@
 1. 只搜索网站根目录下的index.htm(l)内容
 1. 只记录网站有title及description内容的网站，对title及description限制最大长度20和255. 
 
+### 问题
+1. URL有效字符37，按URL10个长度算，37^10=4808584372417849，天文数字
+
 
 

go.mod

@@ -4,6 +4,7 @@ go 1.19
 
 require (
 	github.com/google/uuid v1.3.0
+	golang.org/x/net v0.0.0-20221014081412-f15817d10f9b
 	gorm.io/driver/mysql v1.4.1
 	gorm.io/gorm v1.24.0
 	gorm.io/plugin/dbresolver v1.3.0

go.sum

@@ -7,6 +7,8 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr
 github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU=
+golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 gorm.io/driver/mysql v1.3.2/go.mod h1:ChK6AHbHgDCFZyJp0F+BmVGb06PSIoh9uVYKAlRbb2U=
 gorm.io/driver/mysql v1.4.1 h1:4InA6SOaYtt4yYpV1NF9B2kvUKe9TbvUd1iWrvxnjic=
 gorm.io/driver/mysql v1.4.1/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=

model/domain-model.go

@@ -5,9 +5,7 @@ import "time"
 type Domain struct {
 	Root      string `gorm:"type:varchar(10);primarykey"`
 	CreatedAt time.Time
-	UpdatedAt time.Time
+	UpdatedAt time.Time `gorm:"index"`
-	Success   uint
+	Success   uint64
-	Failure   uint
+	Failure   uint64
-	Current   uint8
-	Status    uint8
 }

model/site-model.go

@@ -3,10 +3,10 @@ package model
 import "time"
 
 type Site struct {
-	URL            string    `gorm:"type:varchar(255);primarykey"`
+	URL         string    `gorm:"type:varchar(255);primarykey"`
-	Title          string    `gorm:"type:varchar(30);not null;"`
+	Title       string    `gorm:"type:varchar(30);not null;"`
-	Description    string    `gorm:"type:varchar(255);not null;"`
+	Description string    `gorm:"type:varchar(255);not null;"`
-	AccessDuration uint      `gorm:"index"`
+	LinkCount   uint      `gorm:"index"`
-	UpdatedAt      time.Time `gorm:"index"`
+	UpdatedAt   time.Time `gorm:"index"`
-	CreatedAt      time.Time
+	CreatedAt   time.Time
 }

service/domain-service.go

@@ -1,31 +0,0 @@
-package service
-
-import (
-	"time"
-
-	"myschools.me/suguo/search/exceptionless"
-	"myschools.me/suguo/search/model"
-	"myschools.me/suguo/search/mysql"
-)
-
-func domainAnalyze(domain *model.Domain) {
-	if domain.Status != 0 && domain.UpdatedAt.After(time.Now().Add(10*time.Minute)) {
-		return
-	}
-
-	db, err := mysql.New()
-	if err != nil {
-		exceptionless.SubmitAppError("domainAnalyze", "mysql.New", nil, err)
-		return
-	}
-
-	if domain.Current >= uint8(250-len(domain.Root)) {
-		domain.Current = 1
-	}
-	domain.Status = 1 //置状态处理中
-	if err := db.Updates(domain).Error; err != nil {
-		exceptionless.SubmitAppError("domainAnalyze", "mysql.Updates", nil, err)
-		return
-	}
-
-}

service/probe-service.go

@@ -1,6 +1,8 @@
 package service
 
 import (
+	"fmt"
+	"math/rand"
 	"time"
 
 	"gorm.io/gorm"
@@ -20,7 +22,7 @@ func Probe() {
 		}
 
 		var domains []*model.Domain
-		if err := db.Find(&domains).Error; err != nil {
+		if err := db.Where("updated_at<?", time.Now().Add(-10*time.Minute)).Find(&domains).Error; err != nil {
 			if err != gorm.ErrRecordNotFound {
 				exceptionless.SubmitAppError("Probe", "mysql.Find", nil, err)
 				continue
@@ -28,7 +30,23 @@ func Probe() {
 		}
 
 		for _, d := range domains {
-			domainAnalyze(d)
+			for i := 0; i < 500; i++ {
+				url := fmt.Sprintf(`https://www.%s.%s`, randSeq(), d.Root)
+				go siteAccess(&url)
+			}
 		}
 	}
 }
+
+var letters = []rune("0123456789abcdefghijklmnopqrstuvwxyz_")
+
+func randSeq() string {
+	rand.Seed(time.Now().Unix())
+
+	b := make([]rune, rand.Int31n(10))
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	for i := range b {
+		b[i] = letters[r.Intn(37)]
+	}
+	return string(b)
+}

service/site-service.go

@@ -1,5 +1,118 @@
 package service
 
-func siteAccess() {
+import (
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
 
+	"golang.org/x/net/html"
+	"gorm.io/gorm/clause"
+	"myschools.me/suguo/search/exceptionless"
+	"myschools.me/suguo/search/model"
+	"myschools.me/suguo/search/mysql"
+)
+
+func siteAccess(url *string) {
+	client := http.Client{Timeout: 3 * time.Second}
+	req, err := http.NewRequest("GET", *url, nil)
+	if err != nil {
+		exceptionless.SubmitAppError("siteAccess", "http.NewRequest", nil, err)
+		return
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		exceptionless.SubmitAppError("siteAccess", "client.Do", nil, err)
+		return
+	}
+
+	defer resp.Body.Close()
+	title := resp.Header.Get("title")
+	if title == "" {
+		return
+	}
+	description := resp.Header.Get("description")
+	if description == "" {
+		return
+	}
+
+	urls := urlAnalysis(resp.Body)
+	site := &model.Site{
+		URL:         *url,
+		Title:       title,
+		Description: description,
+		LinkCount:   uint(len(urls)),
+		UpdatedAt:   time.Time{},
+		CreatedAt:   time.Time{},
+	}
+	db, err := mysql.New()
+	if err != nil {
+		exceptionless.SubmitAppError("siteAccess", "mysql.New", nil, err)
+		return
+	}
+
+	if err := db.Clauses(clause.OnConflict{UpdateAll: true}).Create(site).Error; err != nil {
+		exceptionless.SubmitAppError("siteAccess", "db.Create", nil, err)
+	}
+}
+
+// 从内容中分析返回所有URL链接
+func urlAnalysis(httpBody io.Reader) []string {
+	links := []string{}
+	col := []string{}
+	page := html.NewTokenizer(httpBody)
+	for {
+		tokenType := page.Next()
+		if tokenType == html.ErrorToken {
+			return links
+		}
+		token := page.Token()
+		if tokenType == html.StartTagToken && token.DataAtom.String() == "a" {
+			for _, attr := range token.Attr {
+				if attr.Key == "href" {
+					tl := trimHash(attr.Val)
+					col = append(col, tl)
+					resolv(&links, col)
+				}
+			}
+		}
+	}
+}
+
+// trimHash slices a hash # from the link
+func trimHash(l string) string {
+	if strings.Contains(l, "#") {
+		var index int
+		for n, str := range l {
+			if strconv.QuoteRune(str) == "'#'" {
+				index = n
+				break
+			}
+		}
+		return l[:index]
+	}
+	return l
+}
+
+// check looks to see if a url exits in the slice.
+func check(sl []string, s string) bool {
+	var check bool
+	for _, str := range sl {
+		if str == s {
+			check = true
+			break
+		}
+	}
+	return check
+}
+
+// resolv adds links to the link slice and insures that there is no repetition
+// in our collection.
+func resolv(sl *[]string, ml []string) {
+	for _, str := range ml {
+		if !check(*sl, str) {
+			*sl = append(*sl, str)
+		}
+	}
 }