diff --git a/README.md b/README.md index eb002af..63916e8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ -# 我的搜索(search) +# 我的站点搜索(search) -#### 概述 +### 概述 试写我的搜索引擎试试，基本思想如下： -1. 起字母顺序起 +1. 只搜索国内能访问到的，超时时长在3s内的网站 +1. 只搜索网站根目录下的index.htm(l)内容 +1. 只记录网站有title及description内容的网站，对title及description限制最大长度20和255. + diff --git a/exceptionless/api.go b/exceptionless/api.go new file mode 100644 index 0000000..ac6beb1 --- /dev/null +++ b/exceptionless/api.go @@ -0,0 +1,63 @@ +package exceptionless + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +// Post posts to the Exceptionless Server +func Post(endpoint string, postBody string, authorization string) string { + url := conf.ServerURL + endpoint + var jsonStr = []byte(postBody) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr)) + if err != nil { + return err.Error() + } + req.Header.Set("Authorization", "Bearer "+authorization) + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err.Error() + } + defer resp.Body.Close() + // body, _ := ioutil.ReadAll(resp.Body) + return string(resp.Status) +} + +// GET makes api GET requests +func Get(endpoint string, authorization string) map[string]interface{} { + url := conf.ServerURL + endpoint + + httpClient := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + + if err != nil { + fmt.Println(err) + // return "Error" + } + + req.Header.Add("accept", "application/json") + req.Header.Add("Authorization", "Bearer "+authorization) + + res, err := httpClient.Do(req) + if err != nil { + fmt.Println(err) + // return "Error" + } + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + fmt.Println(err) + // return "Error" + } + + // resp := string(body) + var result map[string]interface{} + json.Unmarshal([]byte(body), &result) + return result +} diff --git a/exceptionless/exceptionless.go b/exceptionless/exceptionless.go new file mode 100644 index 0000000..eaff8e2 --- /dev/null +++ b/exceptionless/exceptionless.go @@ -0,0 +1,217 @@ +package exceptionless + +import ( + "encoding/json" + "fmt" + "math/rand" + "time" + + "github.com/google/uuid" +) + +var config map[string]interface{} = nil + +// Config type defines the client configuration structure +type Config struct { + ApiKey string + ServerURL string + ProgramName string + UpdateSettingsWhenIdleInterval int32 +} + +var conf *Config + +func Init(config *Config) { + conf = config + if conf == nil { + return + } + + if conf.ApiKey != "" && conf.UpdateSettingsWhenIdleInterval > 0 { + poll() + } +} + +// GetConfig returns the project configuration +func GetConfig() map[string]interface{} { + return config +} + +// SubmitEvent sends log events to Exceptionless +func SubmitEvent(eventBody string) string { + if conf.ApiKey == "" { + return "" + } + resp := Post("events", eventBody, conf.ApiKey) + return resp +} + +func poll() { + r := rand.New(rand.NewSource(99)) + c := time.Tick(10 * time.Second) + for _ = range c { + resp := Get("projects/config", conf.ApiKey) + config = resp + jitter := time.Duration(r.Int31n(conf.UpdateSettingsWhenIdleInterval)) * time.Millisecond + time.Sleep(jitter) + } +} + +// Event is the main model for events +type Event struct { + EventType string `json:"type"` + Source string `json:"source,omitempty"` + Date string `json:"date,omitempty"` + Tags []string `json:"tags,omitempty"` + Message string `json:"message,omitempty"` + Geo string `json:"geo,omitempty"` + Value uint `json:"value,omitempty"` + Data map[string]interface{} `json:"data,omitempty"` + ReferenceID string `json:"reference_id,omitempty"` + Count uint `json:"count,omitempty"` +} + +// GetBaseEvent returns an empty Event struct that can be built into any type of event. +func GetBaseEvent(eventType string, message string, date string) Event { + return Event{ + EventType: eventType, + Message: message, + Date: date, + } +} + +// AddSource adds a string value source to an event +func AddSource(event Event, source string) Event { + event.Source = source + return event +} + +// AddTags adds a string array of tags for the event +func AddTags(event Event, tags []string) Event { + event.Tags = tags + return event +} + +// AddGeo adds the lat and long location of the user the event impacted +func AddGeo(event Event, geo string) Event { + event.Geo = geo + return event +} + +// AddValue adds an arbitrary number value to the event +func AddValue(event Event, value uint) Event { + event.Value = value + return event +} + +// AddReferenceID adds an indentifier to later refer to this event +func AddReferenceID(event Event, referenceID uuid.UUID) Event { + event.ReferenceID = referenceID.String() + return event +} + +// AddCount adds a number to help track the number of times the event has occurred +func AddCount(event Event, count uint) Event { + event.Count = count + return event +} + +func AddLogLevel(event Event, level string) Event { + var updatedEvent Event + if event.Data != nil { + event.Data["@level"] = level + updatedEvent = event + } else { + data := map[string]interface{}{} + data["@level"] = level + updatedEvent = AddData(event, data) + } + return updatedEvent +} + +// AddData adds a string mapping to create a data object of additional values +func AddData(event Event, data map[string]interface{}) Event { + event.Data = data + return event +} + +func SubmitAppError(funcName string, proc string, reqid *string, err error) string { + referenceID := uuid.Must(uuid.NewUUID()).String() + + if reqid != nil { + referenceID = *reqid + } + + errorMap := map[string]interface{}{} + errorMap["function"] = funcName + errorMap["type"] = "error" + errorMap["message"] = err.Error() + errorMap["date"] = time.Now().Format(time.RFC3339) + data := map[string]interface{}{} + data["@simple_error"] = errorMap + var event = Event{ + EventType: "error", + Message: fmt.Sprintf("%s - %s", funcName, proc), + Data: data, + ReferenceID: referenceID, + Source: conf.ProgramName, + } + json, err := json.Marshal(event) + if err != nil { + return err.Error() + } + resp := SubmitEvent(string(json)) + return resp +} + +// SubmitError is a convenience wrapper to quickly build and submit an error +func SubmitError(err error) string { + if conf.UpdateSettingsWhenIdleInterval > 0 { + GetConfig() + } + referenceID := uuid.Must(uuid.NewUUID()) + errorMap := map[string]interface{}{} + errorMap["type"] = "error" + errorMap["message"] = err.Error() + errorMap["date"] = time.Now().Format(time.RFC3339) + data := map[string]interface{}{} + data["@simple_error"] = errorMap + var mainEvent = Event{ + EventType: "error", + Message: err.Error(), + Data: data, + ReferenceID: referenceID.String(), + } + json, err := json.Marshal(mainEvent) + if err != nil { + fmt.Println(err) + return err.Error() + } + resp := SubmitEvent(string(json)) + return resp +} + +func SubmitLog(message string, level string) string { + referenceID := uuid.Must(uuid.NewUUID()) + if conf.UpdateSettingsWhenIdleInterval > 0 { + config := GetConfig() + fmt.Println(config) + // We are stripping out info accoring to the config settings + // We would also prevent logs of levels below the log level set by the settings + } + var event Event + date := time.Now().Format(time.RFC3339) + event = GetBaseEvent("log", message, date) + event = AddReferenceID(event, referenceID) + data := map[string]interface{}{} + data["@level"] = level + event = AddData(event, data) + + json, err := json.Marshal(event) + if err != nil { + fmt.Println(err) + return err.Error() + } + resp := SubmitEvent(string(json)) + return resp +} diff --git a/go.mod b/go.mod index bb1c3f3..8c09e11 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,16 @@ -module suguo/search +module myschools.me/suguo/search go 1.19 + +require ( + github.com/google/uuid v1.3.0 + gorm.io/driver/mysql v1.4.1 + gorm.io/gorm v1.24.0 + gorm.io/plugin/dbresolver v1.3.0 +) + +require ( + github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e5fc8a0 --- /dev/null +++ b/go.sum @@ -0,0 +1,18 @@ +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +gorm.io/driver/mysql v1.3.2/go.mod h1:ChK6AHbHgDCFZyJp0F+BmVGb06PSIoh9uVYKAlRbb2U= +gorm.io/driver/mysql v1.4.1 h1:4InA6SOaYtt4yYpV1NF9B2kvUKe9TbvUd1iWrvxnjic= +gorm.io/driver/mysql v1.4.1/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c= +gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.24.0 h1:j/CoiSm6xpRpmzbFJsQHYj+I8bGYWLXVHeYEyyKlF74= +gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/plugin/dbresolver v1.3.0 h1:uFDX3bIuH9Lhj5LY2oyqR/bU6pqWuDgas35NAPF4X3M= +gorm.io/plugin/dbresolver v1.3.0/go.mod h1:Pr7p5+JFlgDaiM6sOrli5olekJD16YRunMyA2S7ZfKk= diff --git a/main.go b/main.go index 06ab7d0..a08e655 100644 --- a/main.go +++ b/main.go @@ -1 +1,44 @@ package main + +import ( + "context" + "os" + "os/signal" + "time" + + "myschools.me/suguo/search/exceptionless" + "myschools.me/suguo/search/model" + "myschools.me/suguo/search/mysql" +) + +func main() { + // 日志初始化 + exceptionless.Init(&exceptionless.Config{ + ApiKey: os.Getenv("exceptionless_apikey"), + ServerURL: os.Getenv("exceptionless_url"), + }) + + //mysql初始化 + mysql.Init(&mysql.Config{ + ConnString: os.Getenv("mysql_dsn"), + ConnMaxLifetime: 1, + MaxIdleConns: 2, + MaxOpenConns: 300, + }) + db, err := mysql.New() + if err != nil { + panic(err) + } + if err := db.AutoMigrate(&model.Site{}); err != nil { + panic(err) + } + + // 服务停止相应 + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + <-c + _, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + exceptionless.SubmitLog("search service shutting down", "info") + os.Exit(0) +} diff --git a/model/site-model.go b/model/site-model.go new file mode 100644 index 0000000..466ad13 --- /dev/null +++ b/model/site-model.go @@ -0,0 +1,11 @@ +package model + +import "time" + +type Site struct { + URL string `gorm:"type:varchar(255);primarykey"` + Title string `gorm:"type:varchar(30);not null;"` + Description string `gorm:"type:varchar(255);not null;"` + UpdatedAt time.Time `gorm:"index"` + CreatedAt time.Time +} diff --git a/mysql/mysql.go b/mysql/mysql.go new file mode 100644 index 0000000..25ee002 --- /dev/null +++ b/mysql/mysql.go @@ -0,0 +1,70 @@ +package mysql + +import ( + "errors" + "time" + + "gorm.io/driver/mysql" + "gorm.io/gorm" + "gorm.io/gorm/logger" + "gorm.io/gorm/schema" + "gorm.io/plugin/dbresolver" +) + +var ( + _db *gorm.DB + _conf *Config +) + +type Config struct { + ConnString string + ConnMaxLifetime int64 //ConnMaxLifetime 最大连接时间，单位：小时 + MaxIdleConns int + MaxOpenConns int +} + +//Init mysql初始化 +func Init(config *Config) { + if config == nil { + config = &Config{ + ConnString: "root:root@tcp(127.0.0.1:3306)/sample?charset=utf8&parseTime=True&loc=Local", + ConnMaxLifetime: 1, + MaxIdleConns: 10, + MaxOpenConns: 100, + } + } + _conf = config +} + +//New 创建实例 +func New() (*gorm.DB, error) { + if _db != nil { + return _db, nil + } + + if _conf == nil { + return nil, errors.New("组件未初始化，请执行Init！") + } + + var err error + _db, err = gorm.Open(mysql.Open(_conf.ConnString), &gorm.Config{ + SkipDefaultTransaction: true, + Logger: logger.Default.LogMode(logger.Silent), + NamingStrategy: schema.NamingStrategy{ + SingularTable: true, + }, + }) + if err != nil { + return nil, err + } + _db.Use( + dbresolver.Register(dbresolver.Config{ + Sources: []gorm.Dialector{mysql.Open(_conf.ConnString)}, + Replicas: []gorm.Dialector{mysql.Open(_conf.ConnString)}, + Policy: dbresolver.RandomPolicy{}, + }).SetConnMaxIdleTime(time.Hour). + SetConnMaxLifetime(time.Duration(_conf.ConnMaxLifetime) * time.Hour). + SetMaxIdleConns(_conf.MaxIdleConns). + SetMaxOpenConns(_conf.MaxOpenConns)) + return _db, nil +}