init

README.md

@@ -1,7 +1,10 @@
-# 我的搜索(search)
+# 我的站点搜索(search)
 
-#### 概述
+### 概述
 试写我的搜索引擎试试，基本思想如下：
-1. 起字母顺序起
+1. 只搜索国内能访问到的，超时时长在3s内的网站
+1. 只搜索网站根目录下的index.htm(l)内容
+1. 只记录网站有title及description内容的网站，对title及description限制最大长度20和255. 
+
 
 

exceptionless/api.go

@@ -0,0 +1,63 @@
+package exceptionless
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+// Post posts to the Exceptionless Server
+func Post(endpoint string, postBody string, authorization string) string {
+	url := conf.ServerURL + endpoint
+	var jsonStr = []byte(postBody)
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
+	if err != nil {
+		return err.Error()
+	}
+	req.Header.Set("Authorization", "Bearer "+authorization)
+	req.Header.Set("Content-Type", "application/json")
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err.Error()
+	}
+	defer resp.Body.Close()
+	// body, _ := ioutil.ReadAll(resp.Body)
+	return string(resp.Status)
+}
+
+// GET makes api GET requests
+func Get(endpoint string, authorization string) map[string]interface{} {
+	url := conf.ServerURL + endpoint
+
+	httpClient := &http.Client{}
+	req, err := http.NewRequest("GET", url, nil)
+
+	if err != nil {
+		fmt.Println(err)
+		// return "Error"
+	}
+
+	req.Header.Add("accept", "application/json")
+	req.Header.Add("Authorization", "Bearer "+authorization)
+
+	res, err := httpClient.Do(req)
+	if err != nil {
+		fmt.Println(err)
+		// return "Error"
+	}
+	defer res.Body.Close()
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		fmt.Println(err)
+		// return "Error"
+	}
+
+	// resp := string(body)
+	var result map[string]interface{}
+	json.Unmarshal([]byte(body), &result)
+	return result
+}

exceptionless/exceptionless.go

@@ -0,0 +1,217 @@
+package exceptionless
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+var config map[string]interface{} = nil
+
+// Config type defines the client configuration structure
+type Config struct {
+	ApiKey                         string
+	ServerURL                      string
+	ProgramName                    string
+	UpdateSettingsWhenIdleInterval int32
+}
+
+var conf *Config
+
+func Init(config *Config) {
+	conf = config
+	if conf == nil {
+		return
+	}
+
+	if conf.ApiKey != "" && conf.UpdateSettingsWhenIdleInterval > 0 {
+		poll()
+	}
+}
+
+// GetConfig returns the project configuration
+func GetConfig() map[string]interface{} {
+	return config
+}
+
+// SubmitEvent sends log events to Exceptionless
+func SubmitEvent(eventBody string) string {
+	if conf.ApiKey == "" {
+		return ""
+	}
+	resp := Post("events", eventBody, conf.ApiKey)
+	return resp
+}
+
+func poll() {
+	r := rand.New(rand.NewSource(99))
+	c := time.Tick(10 * time.Second)
+	for _ = range c {
+		resp := Get("projects/config", conf.ApiKey)
+		config = resp
+		jitter := time.Duration(r.Int31n(conf.UpdateSettingsWhenIdleInterval)) * time.Millisecond
+		time.Sleep(jitter)
+	}
+}
+
+// Event is the main model for events
+type Event struct {
+	EventType   string                 `json:"type"`
+	Source      string                 `json:"source,omitempty"`
+	Date        string                 `json:"date,omitempty"`
+	Tags        []string               `json:"tags,omitempty"`
+	Message     string                 `json:"message,omitempty"`
+	Geo         string                 `json:"geo,omitempty"`
+	Value       uint                   `json:"value,omitempty"`
+	Data        map[string]interface{} `json:"data,omitempty"`
+	ReferenceID string                 `json:"reference_id,omitempty"`
+	Count       uint                   `json:"count,omitempty"`
+}
+
+// GetBaseEvent returns an empty Event struct that can be built into any type of event.
+func GetBaseEvent(eventType string, message string, date string) Event {
+	return Event{
+		EventType: eventType,
+		Message:   message,
+		Date:      date,
+	}
+}
+
+// AddSource adds a string value source to an event
+func AddSource(event Event, source string) Event {
+	event.Source = source
+	return event
+}
+
+// AddTags adds a string array of tags for the event
+func AddTags(event Event, tags []string) Event {
+	event.Tags = tags
+	return event
+}
+
+// AddGeo adds the lat and long location of the user the event impacted
+func AddGeo(event Event, geo string) Event {
+	event.Geo = geo
+	return event
+}
+
+// AddValue adds an arbitrary number value to the event
+func AddValue(event Event, value uint) Event {
+	event.Value = value
+	return event
+}
+
+// AddReferenceID adds an indentifier to later refer to this event
+func AddReferenceID(event Event, referenceID uuid.UUID) Event {
+	event.ReferenceID = referenceID.String()
+	return event
+}
+
+// AddCount adds a number to help track the number of times the event has occurred
+func AddCount(event Event, count uint) Event {
+	event.Count = count
+	return event
+}
+
+func AddLogLevel(event Event, level string) Event {
+	var updatedEvent Event
+	if event.Data != nil {
+		event.Data["@level"] = level
+		updatedEvent = event
+	} else {
+		data := map[string]interface{}{}
+		data["@level"] = level
+		updatedEvent = AddData(event, data)
+	}
+	return updatedEvent
+}
+
+// AddData adds a string mapping to create a data object of additional values
+func AddData(event Event, data map[string]interface{}) Event {
+	event.Data = data
+	return event
+}
+
+func SubmitAppError(funcName string, proc string, reqid *string, err error) string {
+	referenceID := uuid.Must(uuid.NewUUID()).String()
+
+	if reqid != nil {
+		referenceID = *reqid
+	}
+
+	errorMap := map[string]interface{}{}
+	errorMap["function"] = funcName
+	errorMap["type"] = "error"
+	errorMap["message"] = err.Error()
+	errorMap["date"] = time.Now().Format(time.RFC3339)
+	data := map[string]interface{}{}
+	data["@simple_error"] = errorMap
+	var event = Event{
+		EventType:   "error",
+		Message:     fmt.Sprintf("%s - %s", funcName, proc),
+		Data:        data,
+		ReferenceID: referenceID,
+		Source:      conf.ProgramName,
+	}
+	json, err := json.Marshal(event)
+	if err != nil {
+		return err.Error()
+	}
+	resp := SubmitEvent(string(json))
+	return resp
+}
+
+// SubmitError is a convenience wrapper to quickly build and submit an error
+func SubmitError(err error) string {
+	if conf.UpdateSettingsWhenIdleInterval > 0 {
+		GetConfig()
+	}
+	referenceID := uuid.Must(uuid.NewUUID())
+	errorMap := map[string]interface{}{}
+	errorMap["type"] = "error"
+	errorMap["message"] = err.Error()
+	errorMap["date"] = time.Now().Format(time.RFC3339)
+	data := map[string]interface{}{}
+	data["@simple_error"] = errorMap
+	var mainEvent = Event{
+		EventType:   "error",
+		Message:     err.Error(),
+		Data:        data,
+		ReferenceID: referenceID.String(),
+	}
+	json, err := json.Marshal(mainEvent)
+	if err != nil {
+		fmt.Println(err)
+		return err.Error()
+	}
+	resp := SubmitEvent(string(json))
+	return resp
+}
+
+func SubmitLog(message string, level string) string {
+	referenceID := uuid.Must(uuid.NewUUID())
+	if conf.UpdateSettingsWhenIdleInterval > 0 {
+		config := GetConfig()
+		fmt.Println(config)
+		//	We are stripping out info accoring to the config settings
+		//	We would also prevent logs of levels below the log level set by the settings
+	}
+	var event Event
+	date := time.Now().Format(time.RFC3339)
+	event = GetBaseEvent("log", message, date)
+	event = AddReferenceID(event, referenceID)
+	data := map[string]interface{}{}
+	data["@level"] = level
+	event = AddData(event, data)
+
+	json, err := json.Marshal(event)
+	if err != nil {
+		fmt.Println(err)
+		return err.Error()
+	}
+	resp := SubmitEvent(string(json))
+	return resp
+}

go.mod

@@ -1,3 +1,16 @@
-module suguo/search
+module myschools.me/suguo/search
 
 go 1.19
+
+require (
+	github.com/google/uuid v1.3.0
+	gorm.io/driver/mysql v1.4.1
+	gorm.io/gorm v1.24.0
+	gorm.io/plugin/dbresolver v1.3.0
+)
+
+require (
+	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+)

go.sum

@@ -0,0 +1,18 @@
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+gorm.io/driver/mysql v1.3.2/go.mod h1:ChK6AHbHgDCFZyJp0F+BmVGb06PSIoh9uVYKAlRbb2U=
+gorm.io/driver/mysql v1.4.1 h1:4InA6SOaYtt4yYpV1NF9B2kvUKe9TbvUd1iWrvxnjic=
+gorm.io/driver/mysql v1.4.1/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
+gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
+gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
+gorm.io/gorm v1.24.0 h1:j/CoiSm6xpRpmzbFJsQHYj+I8bGYWLXVHeYEyyKlF74=
+gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
+gorm.io/plugin/dbresolver v1.3.0 h1:uFDX3bIuH9Lhj5LY2oyqR/bU6pqWuDgas35NAPF4X3M=
+gorm.io/plugin/dbresolver v1.3.0/go.mod h1:Pr7p5+JFlgDaiM6sOrli5olekJD16YRunMyA2S7ZfKk=

main.go

@@ -1 +1,44 @@
 package main
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"time"
+
+	"myschools.me/suguo/search/exceptionless"
+	"myschools.me/suguo/search/model"
+	"myschools.me/suguo/search/mysql"
+)
+
+func main() {
+	// 日志初始化
+	exceptionless.Init(&exceptionless.Config{
+		ApiKey:    os.Getenv("exceptionless_apikey"),
+		ServerURL: os.Getenv("exceptionless_url"),
+	})
+
+	//mysql初始化
+	mysql.Init(&mysql.Config{
+		ConnString:      os.Getenv("mysql_dsn"),
+		ConnMaxLifetime: 1,
+		MaxIdleConns:    2,
+		MaxOpenConns:    300,
+	})
+	db, err := mysql.New()
+	if err != nil {
+		panic(err)
+	}
+	if err := db.AutoMigrate(&model.Site{}); err != nil {
+		panic(err)
+	}
+
+	// 服务停止相应
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	<-c
+	_, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	exceptionless.SubmitLog("search service shutting down", "info")
+	os.Exit(0)
+}

model/site-model.go

@@ -0,0 +1,11 @@
+package model
+
+import "time"
+
+type Site struct {
+	URL         string    `gorm:"type:varchar(255);primarykey"`
+	Title       string    `gorm:"type:varchar(30);not null;"`
+	Description string    `gorm:"type:varchar(255);not null;"`
+	UpdatedAt   time.Time `gorm:"index"`
+	CreatedAt   time.Time
+}

mysql/mysql.go

@@ -0,0 +1,70 @@
+package mysql
+
+import (
+	"errors"
+	"time"
+
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"gorm.io/gorm/schema"
+	"gorm.io/plugin/dbresolver"
+)
+
+var (
+	_db   *gorm.DB
+	_conf *Config
+)
+
+type Config struct {
+	ConnString      string
+	ConnMaxLifetime int64 //ConnMaxLifetime 最大连接时间，单位：小时
+	MaxIdleConns    int
+	MaxOpenConns    int
+}
+
+//Init mysql初始化
+func Init(config *Config) {
+	if config == nil {
+		config = &Config{
+			ConnString:      "root:root@tcp(127.0.0.1:3306)/sample?charset=utf8&parseTime=True&loc=Local",
+			ConnMaxLifetime: 1,
+			MaxIdleConns:    10,
+			MaxOpenConns:    100,
+		}
+	}
+	_conf = config
+}
+
+//New 创建实例
+func New() (*gorm.DB, error) {
+	if _db != nil {
+		return _db, nil
+	}
+
+	if _conf == nil {
+		return nil, errors.New("组件未初始化，请执行Init！")
+	}
+
+	var err error
+	_db, err = gorm.Open(mysql.Open(_conf.ConnString), &gorm.Config{
+		SkipDefaultTransaction: true,
+		Logger:                 logger.Default.LogMode(logger.Silent),
+		NamingStrategy: schema.NamingStrategy{
+			SingularTable: true,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	_db.Use(
+		dbresolver.Register(dbresolver.Config{
+			Sources:  []gorm.Dialector{mysql.Open(_conf.ConnString)},
+			Replicas: []gorm.Dialector{mysql.Open(_conf.ConnString)},
+			Policy:   dbresolver.RandomPolicy{},
+		}).SetConnMaxIdleTime(time.Hour).
+			SetConnMaxLifetime(time.Duration(_conf.ConnMaxLifetime) * time.Hour).
+			SetMaxIdleConns(_conf.MaxIdleConns).
+			SetMaxOpenConns(_conf.MaxOpenConns))
+	return _db, nil
+}