init
This commit is contained in:
parent
e8722e6df5
commit
b7dfefbf1c
|
|
@ -6,5 +6,8 @@
|
|||
1. 只搜索网站根目录下的index.htm(l)内容
|
||||
1. 只记录网站有title及description内容的网站，对title及description限制最大长度20和255.
|
||||
|
||||
### 问题
|
||||
1. URL有效字符37，按URL10个长度算，37^10=4808584372417849，天文数字
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
1
go.mod
1
go.mod
|
|
@ -4,6 +4,7 @@ go 1.19
|
|||
|
||||
require (
|
||||
github.com/google/uuid v1.3.0
|
||||
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b
|
||||
gorm.io/driver/mysql v1.4.1
|
||||
gorm.io/gorm v1.24.0
|
||||
gorm.io/plugin/dbresolver v1.3.0
|
||||
|
|
|
|||
2
go.sum
2
go.sum
|
|
@ -7,6 +7,8 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr
|
|||
github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU=
|
||||
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
gorm.io/driver/mysql v1.3.2/go.mod h1:ChK6AHbHgDCFZyJp0F+BmVGb06PSIoh9uVYKAlRbb2U=
|
||||
gorm.io/driver/mysql v1.4.1 h1:4InA6SOaYtt4yYpV1NF9B2kvUKe9TbvUd1iWrvxnjic=
|
||||
gorm.io/driver/mysql v1.4.1/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
|
||||
|
|
|
|||
|
|
@ -5,9 +5,7 @@ import "time"
|
|||
type Domain struct {
|
||||
Root string `gorm:"type:varchar(10);primarykey"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
Success uint
|
||||
Failure uint
|
||||
Current uint8
|
||||
Status uint8
|
||||
UpdatedAt time.Time `gorm:"index"`
|
||||
Success uint64
|
||||
Failure uint64
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ package model
|
|||
import "time"
|
||||
|
||||
type Site struct {
|
||||
URL string `gorm:"type:varchar(255);primarykey"`
|
||||
Title string `gorm:"type:varchar(30);not null;"`
|
||||
Description string `gorm:"type:varchar(255);not null;"`
|
||||
AccessDuration uint `gorm:"index"`
|
||||
UpdatedAt time.Time `gorm:"index"`
|
||||
CreatedAt time.Time
|
||||
URL string `gorm:"type:varchar(255);primarykey"`
|
||||
Title string `gorm:"type:varchar(30);not null;"`
|
||||
Description string `gorm:"type:varchar(255);not null;"`
|
||||
LinkCount uint `gorm:"index"`
|
||||
UpdatedAt time.Time `gorm:"index"`
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,31 +0,0 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"myschools.me/suguo/search/exceptionless"
|
||||
"myschools.me/suguo/search/model"
|
||||
"myschools.me/suguo/search/mysql"
|
||||
)
|
||||
|
||||
func domainAnalyze(domain *model.Domain) {
|
||||
if domain.Status != 0 && domain.UpdatedAt.After(time.Now().Add(10*time.Minute)) {
|
||||
return
|
||||
}
|
||||
|
||||
db, err := mysql.New()
|
||||
if err != nil {
|
||||
exceptionless.SubmitAppError("domainAnalyze", "mysql.New", nil, err)
|
||||
return
|
||||
}
|
||||
|
||||
if domain.Current >= uint8(250-len(domain.Root)) {
|
||||
domain.Current = 1
|
||||
}
|
||||
domain.Status = 1 //置状态处理中
|
||||
if err := db.Updates(domain).Error; err != nil {
|
||||
exceptionless.SubmitAppError("domainAnalyze", "mysql.Updates", nil, err)
|
||||
return
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
package service
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
|
@ -20,7 +22,7 @@ func Probe() {
|
|||
}
|
||||
|
||||
var domains []*model.Domain
|
||||
if err := db.Find(&domains).Error; err != nil {
|
||||
if err := db.Where("updated_at<?", time.Now().Add(-10*time.Minute)).Find(&domains).Error; err != nil {
|
||||
if err != gorm.ErrRecordNotFound {
|
||||
exceptionless.SubmitAppError("Probe", "mysql.Find", nil, err)
|
||||
continue
|
||||
|
|
@ -28,7 +30,23 @@ func Probe() {
|
|||
}
|
||||
|
||||
for _, d := range domains {
|
||||
domainAnalyze(d)
|
||||
for i := 0; i < 500; i++ {
|
||||
url := fmt.Sprintf(`https://www.%s.%s`, randSeq(), d.Root)
|
||||
go siteAccess(&url)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var letters = []rune("0123456789abcdefghijklmnopqrstuvwxyz_")
|
||||
|
||||
func randSeq() string {
|
||||
rand.Seed(time.Now().Unix())
|
||||
|
||||
b := make([]rune, rand.Int31n(10))
|
||||
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
for i := range b {
|
||||
b[i] = letters[r.Intn(37)]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,118 @@
|
|||
package service
|
||||
|
||||
func siteAccess() {
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
"gorm.io/gorm/clause"
|
||||
"myschools.me/suguo/search/exceptionless"
|
||||
"myschools.me/suguo/search/model"
|
||||
"myschools.me/suguo/search/mysql"
|
||||
)
|
||||
|
||||
func siteAccess(url *string) {
|
||||
client := http.Client{Timeout: 3 * time.Second}
|
||||
req, err := http.NewRequest("GET", *url, nil)
|
||||
if err != nil {
|
||||
exceptionless.SubmitAppError("siteAccess", "http.NewRequest", nil, err)
|
||||
return
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
exceptionless.SubmitAppError("siteAccess", "client.Do", nil, err)
|
||||
return
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
title := resp.Header.Get("title")
|
||||
if title == "" {
|
||||
return
|
||||
}
|
||||
description := resp.Header.Get("description")
|
||||
if description == "" {
|
||||
return
|
||||
}
|
||||
|
||||
urls := urlAnalysis(resp.Body)
|
||||
site := &model.Site{
|
||||
URL: *url,
|
||||
Title: title,
|
||||
Description: description,
|
||||
LinkCount: uint(len(urls)),
|
||||
UpdatedAt: time.Time{},
|
||||
CreatedAt: time.Time{},
|
||||
}
|
||||
db, err := mysql.New()
|
||||
if err != nil {
|
||||
exceptionless.SubmitAppError("siteAccess", "mysql.New", nil, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := db.Clauses(clause.OnConflict{UpdateAll: true}).Create(site).Error; err != nil {
|
||||
exceptionless.SubmitAppError("siteAccess", "db.Create", nil, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 从内容中分析返回所有URL链接
|
||||
func urlAnalysis(httpBody io.Reader) []string {
|
||||
links := []string{}
|
||||
col := []string{}
|
||||
page := html.NewTokenizer(httpBody)
|
||||
for {
|
||||
tokenType := page.Next()
|
||||
if tokenType == html.ErrorToken {
|
||||
return links
|
||||
}
|
||||
token := page.Token()
|
||||
if tokenType == html.StartTagToken && token.DataAtom.String() == "a" {
|
||||
for _, attr := range token.Attr {
|
||||
if attr.Key == "href" {
|
||||
tl := trimHash(attr.Val)
|
||||
col = append(col, tl)
|
||||
resolv(&links, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// trimHash slices a hash # from the link
|
||||
func trimHash(l string) string {
|
||||
if strings.Contains(l, "#") {
|
||||
var index int
|
||||
for n, str := range l {
|
||||
if strconv.QuoteRune(str) == "'#'" {
|
||||
index = n
|
||||
break
|
||||
}
|
||||
}
|
||||
return l[:index]
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// check looks to see if a url exits in the slice.
|
||||
func check(sl []string, s string) bool {
|
||||
var check bool
|
||||
for _, str := range sl {
|
||||
if str == s {
|
||||
check = true
|
||||
break
|
||||
}
|
||||
}
|
||||
return check
|
||||
}
|
||||
|
||||
// resolv adds links to the link slice and insures that there is no repetition
|
||||
// in our collection.
|
||||
func resolv(sl *[]string, ml []string) {
|
||||
for _, str := range ml {
|
||||
if !check(*sl, str) {
|
||||
*sl = append(*sl, str)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue