suguo
/
search
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

init

This commit is contained in:
suguo.yao 2022-10-17 12:14:39 +08:00
parent 75fab78540
commit 5ff5a33222
8 changed files with 442 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -1,7 +1,10 @@
# 我的搜索(search)
# 我的站点搜索(search)
#### 概述
### 概述
试写我的搜索引擎试试，基本思想如下：
1. 起字母顺序起
1. 只搜索国内能访问到的，超时时长在3s内的网站
1. 只搜索网站根目录下的index.htm(l)内容
1. 只记录网站有title及description内容的网站，对title及description限制最大长度20和255.

63
exceptionless/api.go Normal file
View File

@ -0,0 +1,63 @@
package exceptionless
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
)
// Post posts to the Exceptionless Server
func Post(endpoint string, postBody string, authorization string) string {
url := conf.ServerURL + endpoint
var jsonStr = []byte(postBody)
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
if err != nil {
return err.Error()
}
req.Header.Set("Authorization", "Bearer "+authorization)
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return err.Error()
}
defer resp.Body.Close()
// body, _ := ioutil.ReadAll(resp.Body)
return string(resp.Status)
}
// GET makes api GET requests
func Get(endpoint string, authorization string) map[string]interface{} {
url := conf.ServerURL + endpoint
httpClient := &http.Client{}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
fmt.Println(err)
// return "Error"
}
req.Header.Add("accept", "application/json")
req.Header.Add("Authorization", "Bearer "+authorization)
res, err := httpClient.Do(req)
if err != nil {
fmt.Println(err)
// return "Error"
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
// return "Error"
}
// resp := string(body)
var result map[string]interface{}
json.Unmarshal([]byte(body), &result)
return result
}

217
exceptionless/exceptionless.go Normal file
View File

@ -0,0 +1,217 @@
package exceptionless
import (
"encoding/json"
"fmt"
"math/rand"
"time"
"github.com/google/uuid"
)
var config map[string]interface{} = nil
// Config type defines the client configuration structure
type Config struct {
ApiKey string
ServerURL string
ProgramName string
UpdateSettingsWhenIdleInterval int32
}
var conf *Config
func Init(config *Config) {
conf = config
if conf == nil {
return
}
if conf.ApiKey != "" && conf.UpdateSettingsWhenIdleInterval > 0 {
poll()
}
}
// GetConfig returns the project configuration
func GetConfig() map[string]interface{} {
return config
}
// SubmitEvent sends log events to Exceptionless
func SubmitEvent(eventBody string) string {
if conf.ApiKey == "" {
return ""
}
resp := Post("events", eventBody, conf.ApiKey)
return resp
}
func poll() {
r := rand.New(rand.NewSource(99))
c := time.Tick(10 * time.Second)
for _ = range c {
resp := Get("projects/config", conf.ApiKey)
config = resp
jitter := time.Duration(r.Int31n(conf.UpdateSettingsWhenIdleInterval)) * time.Millisecond
time.Sleep(jitter)
}
}
// Event is the main model for events
type Event struct {
EventType string `json:"type"`
Source string `json:"source,omitempty"`
Date string `json:"date,omitempty"`
Tags []string `json:"tags,omitempty"`
Message string `json:"message,omitempty"`
Geo string `json:"geo,omitempty"`
Value uint `json:"value,omitempty"`
Data map[string]interface{} `json:"data,omitempty"`
ReferenceID string `json:"reference_id,omitempty"`
Count uint `json:"count,omitempty"`
}
// GetBaseEvent returns an empty Event struct that can be built into any type of event.
func GetBaseEvent(eventType string, message string, date string) Event {
return Event{
EventType: eventType,
Message: message,
Date: date,
}
}
// AddSource adds a string value source to an event
func AddSource(event Event, source string) Event {
event.Source = source
return event
}
// AddTags adds a string array of tags for the event
func AddTags(event Event, tags []string) Event {
event.Tags = tags
return event
}
// AddGeo adds the lat and long location of the user the event impacted
func AddGeo(event Event, geo string) Event {
event.Geo = geo
return event
}
// AddValue adds an arbitrary number value to the event
func AddValue(event Event, value uint) Event {
event.Value = value
return event
}
// AddReferenceID adds an indentifier to later refer to this event
func AddReferenceID(event Event, referenceID uuid.UUID) Event {
event.ReferenceID = referenceID.String()
return event
}
// AddCount adds a number to help track the number of times the event has occurred
func AddCount(event Event, count uint) Event {
event.Count = count
return event
}
func AddLogLevel(event Event, level string) Event {
var updatedEvent Event
if event.Data != nil {
event.Data["@level"] = level
updatedEvent = event
} else {
data := map[string]interface{}{}
data["@level"] = level
updatedEvent = AddData(event, data)
}
return updatedEvent
}
// AddData adds a string mapping to create a data object of additional values
func AddData(event Event, data map[string]interface{}) Event {
event.Data = data
return event
}
func SubmitAppError(funcName string, proc string, reqid *string, err error) string {
referenceID := uuid.Must(uuid.NewUUID()).String()
if reqid != nil {
referenceID = *reqid
}
errorMap := map[string]interface{}{}
errorMap["function"] = funcName
errorMap["type"] = "error"
errorMap["message"] = err.Error()
errorMap["date"] = time.Now().Format(time.RFC3339)
data := map[string]interface{}{}
data["@simple_error"] = errorMap
var event = Event{
EventType: "error",
Message: fmt.Sprintf("%s - %s", funcName, proc),
Data: data,
ReferenceID: referenceID,
Source: conf.ProgramName,
}
json, err := json.Marshal(event)
if err != nil {
return err.Error()
}
resp := SubmitEvent(string(json))
return resp
}
// SubmitError is a convenience wrapper to quickly build and submit an error
func SubmitError(err error) string {
if conf.UpdateSettingsWhenIdleInterval > 0 {
GetConfig()
}
referenceID := uuid.Must(uuid.NewUUID())
errorMap := map[string]interface{}{}
errorMap["type"] = "error"
errorMap["message"] = err.Error()
errorMap["date"] = time.Now().Format(time.RFC3339)
data := map[string]interface{}{}
data["@simple_error"] = errorMap
var mainEvent = Event{
EventType: "error",
Message: err.Error(),
Data: data,
ReferenceID: referenceID.String(),
}
json, err := json.Marshal(mainEvent)
if err != nil {
fmt.Println(err)
return err.Error()
}
resp := SubmitEvent(string(json))
return resp
}
func SubmitLog(message string, level string) string {
referenceID := uuid.Must(uuid.NewUUID())
if conf.UpdateSettingsWhenIdleInterval > 0 {
config := GetConfig()
fmt.Println(config)
// We are stripping out info accoring to the config settings
// We would also prevent logs of levels below the log level set by the settings
}
var event Event
date := time.Now().Format(time.RFC3339)
event = GetBaseEvent("log", message, date)
event = AddReferenceID(event, referenceID)
data := map[string]interface{}{}
data["@level"] = level
event = AddData(event, data)
json, err := json.Marshal(event)
if err != nil {
fmt.Println(err)
return err.Error()
}
resp := SubmitEvent(string(json))
return resp
}

15
go.mod
View File

@ -1,3 +1,16 @@
module suguo/search
module myschools.me/suguo/search
go 1.19
require (
github.com/google/uuid v1.3.0
gorm.io/driver/mysql v1.4.1
gorm.io/gorm v1.24.0
gorm.io/plugin/dbresolver v1.3.0
)
require (
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
)

18
go.sum Normal file
View File

@ -0,0 +1,18 @@
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
gorm.io/driver/mysql v1.3.2/go.mod h1:ChK6AHbHgDCFZyJp0F+BmVGb06PSIoh9uVYKAlRbb2U=
gorm.io/driver/mysql v1.4.1 h1:4InA6SOaYtt4yYpV1NF9B2kvUKe9TbvUd1iWrvxnjic=
gorm.io/driver/mysql v1.4.1/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c=
gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
gorm.io/gorm v1.24.0 h1:j/CoiSm6xpRpmzbFJsQHYj+I8bGYWLXVHeYEyyKlF74=
gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
gorm.io/plugin/dbresolver v1.3.0 h1:uFDX3bIuH9Lhj5LY2oyqR/bU6pqWuDgas35NAPF4X3M=
gorm.io/plugin/dbresolver v1.3.0/go.mod h1:Pr7p5+JFlgDaiM6sOrli5olekJD16YRunMyA2S7ZfKk=

43
main.go
View File

@ -1 +1,44 @@
package main
import (
"context"
"os"
"os/signal"
"time"
"myschools.me/suguo/search/exceptionless"
"myschools.me/suguo/search/model"
"myschools.me/suguo/search/mysql"
)
func main() {
// 日志初始化
exceptionless.Init(&exceptionless.Config{
ApiKey: os.Getenv("exceptionless_apikey"),
ServerURL: os.Getenv("exceptionless_url"),
})
//mysql初始化
mysql.Init(&mysql.Config{
ConnString: os.Getenv("mysql_dsn"),
ConnMaxLifetime: 1,
MaxIdleConns: 2,
MaxOpenConns: 300,
})
db, err := mysql.New()
if err != nil {
panic(err)
}
if err := db.AutoMigrate(&model.Site{}); err != nil {
panic(err)
}
// 服务停止相应
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
<-c
_, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
exceptionless.SubmitLog("search service shutting down", "info")
os.Exit(0)
}

11
model/site-model.go Normal file
View File

@ -0,0 +1,11 @@
package model
import "time"
type Site struct {
URL string `gorm:"type:varchar(255);primarykey"`
Title string `gorm:"type:varchar(30);not null;"`
Description string `gorm:"type:varchar(255);not null;"`
UpdatedAt time.Time `gorm:"index"`
CreatedAt time.Time
}

70
mysql/mysql.go Normal file
View File

@ -0,0 +1,70 @@
package mysql
import (
"errors"
"time"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"gorm.io/gorm/schema"
"gorm.io/plugin/dbresolver"
)
var (
_db *gorm.DB
_conf *Config
)
type Config struct {
ConnString string
ConnMaxLifetime int64 //ConnMaxLifetime 最大连接时间，单位：小时
MaxIdleConns int
MaxOpenConns int
}
//Init mysql初始化
func Init(config *Config) {
if config == nil {
config = &Config{
ConnString: "root:root@tcp(127.0.0.1:3306)/sample?charset=utf8&parseTime=True&loc=Local",
ConnMaxLifetime: 1,
MaxIdleConns: 10,
MaxOpenConns: 100,
}
}
_conf = config
}
//New 创建实例
func New() (*gorm.DB, error) {
if _db != nil {
return _db, nil
}
if _conf == nil {
return nil, errors.New("组件未初始化，请执行Init！")
}
var err error
_db, err = gorm.Open(mysql.Open(_conf.ConnString), &gorm.Config{
SkipDefaultTransaction: true,
Logger: logger.Default.LogMode(logger.Silent),
NamingStrategy: schema.NamingStrategy{
SingularTable: true,
},
})
if err != nil {
return nil, err
}
_db.Use(
dbresolver.Register(dbresolver.Config{
Sources: []gorm.Dialector{mysql.Open(_conf.ConnString)},
Replicas: []gorm.Dialector{mysql.Open(_conf.ConnString)},
Policy: dbresolver.RandomPolicy{},
}).SetConnMaxIdleTime(time.Hour).
SetConnMaxLifetime(time.Duration(_conf.ConnMaxLifetime) * time.Hour).
SetMaxIdleConns(_conf.MaxIdleConns).
SetMaxOpenConns(_conf.MaxOpenConns))
return _db, nil
}