refactor(engine): Redis client & disk related functions
sundowndev
parent
faeb2043a9
commit
ee9c15e52c
|
|
@ -1 +1,9 @@
|
|||
package engine
|
||||
|
||||
// GetFilesFromDir walks through a directory and returns
|
||||
// file paths contained inside.
|
||||
func GetFilesFromDir(dir string) (files []string) {
|
||||
// is dir?
|
||||
// is readable/exists?
|
||||
return files
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,13 +29,11 @@ func NewRedisClient(addr, port string) (*RedisClient, error) {
|
|||
}
|
||||
|
||||
// AddFile index a file
|
||||
func (c *RedisClient) AddFile(key string, content string) error {
|
||||
words := strings.Split(content, " ")
|
||||
|
||||
for _, v := range words {
|
||||
if err := c.conn.ZAdd(strings.ToLower(v), &redis.Z{
|
||||
Score: float64(1),
|
||||
Member: key,
|
||||
func (c *RedisClient) AddFile(file, content string) error {
|
||||
for _, v := range GetWordsFromText(content) {
|
||||
if err := c.conn.ZAdd(file, &redis.Z{
|
||||
Score: float64(CountWord(content, v)),
|
||||
Member: strings.ToLower(v),
|
||||
}).Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -44,20 +42,19 @@ func (c *RedisClient) AddFile(key string, content string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// ZRevRange search for a key
|
||||
func (c *RedisClient) ZRevRange(key string) ([]string, error) {
|
||||
// GetKey search for a key
|
||||
func (c *RedisClient) GetKey(key string) ([]string, error) {
|
||||
return c.conn.ZRevRange(key, 0, -1).Result()
|
||||
}
|
||||
|
||||
// GetKey returns a key value
|
||||
func (c *RedisClient) GetKey(key string) (string, error) {
|
||||
return c.conn.Get(key).Result()
|
||||
// GetScore get score of element
|
||||
func (c *RedisClient) GetScore(key, member string) float64 {
|
||||
return c.conn.ZScore(key, member).Val()
|
||||
}
|
||||
|
||||
// GetAllKeys returns a key value
|
||||
func (c *RedisClient) GetAllKeys() (keys []string, err error) {
|
||||
keys, _, err = c.conn.Scan(0, "*", 15).Result()
|
||||
|
||||
keys, _, err = c.conn.Scan(0, "*", 35).Result()
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ const tabPaddingChar = ' '
|
|||
// QueryResult ...
|
||||
type QueryResult struct {
|
||||
File string
|
||||
Count int
|
||||
Count float64
|
||||
FirstMatch string
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,44 @@
|
|||
package engine
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CountWord returns the number of
|
||||
// occurence for a word in a given text.
|
||||
func CountWord(text, word string) int {
|
||||
count := 0
|
||||
|
||||
func CountWord(text, word string) (count int) {
|
||||
for _, v := range strings.Split(text, " ") {
|
||||
if v == word {
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
return count
|
||||
return
|
||||
}
|
||||
|
||||
// GetWordsFromText returns an array of words
|
||||
func GetWordsFromText(text string) (words []string) {
|
||||
scanner := bufio.NewScanner(bytes.NewBufferString(text))
|
||||
scanner.Split(bufio.ScanWords)
|
||||
for scanner.Scan() {
|
||||
words = append(words, strings.ToLower(scanner.Text()))
|
||||
}
|
||||
|
||||
return words
|
||||
}
|
||||
|
||||
// IsTextFile returns whever a file is a
|
||||
// reconized text file or not.
|
||||
func IsTextFile(file []byte) bool {
|
||||
contentType := http.DetectContentType(file)
|
||||
|
||||
return strings.Index(contentType, "text/plain") > -1
|
||||
}
|
||||
|
||||
// GetFirstMatchLine returns the first line to match the given word.
|
||||
func GetFirstMatchLine(data string, word string) string {
|
||||
return "..."
|
||||
}
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -3,8 +3,8 @@ module github.com/sundowndev/go-search
|
|||
go 1.13
|
||||
|
||||
require (
|
||||
github.com/evilsocket/islazy v1.10.6 // indirect
|
||||
github.com/go-redis/redis/v7 v7.2.0
|
||||
github.com/spf13/cobra v1.0.0
|
||||
github.com/stretchr/testify v1.2.2
|
||||
gopkg.in/yaml.v2 v2.2.4
|
||||
)
|
||||
|
|
|
|||
9
go.sum
9
go.sum
|
|
@ -18,14 +18,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
|||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/evilsocket/islazy v1.10.6 h1:MFq000a1ByoumoJWlytqg0qon0KlBeUfPsDjY0hK0bo=
|
||||
github.com/evilsocket/islazy v1.10.6/go.mod h1:OrwQGYg3DuZvXUfmH+KIZDjwTCbrjy48T24TUpGqVVw=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U=
|
||||
github.com/go-redis/redis/v7 v7.2.0 h1:CrCexy/jYWZjW0AyVoHlcJUeZN19VWlbepTh1Vq6dJs=
|
||||
github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
|
|
@ -36,6 +34,7 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
|
|||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
|
|
@ -54,8 +53,10 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
|
|||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
|
|
@ -98,7 +99,6 @@ github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/y
|
|||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/sundowndev/phoneinfoga v1.10.12 h1:lXHpltmy8RD/JlJmTdYqnwwMxqskoOR74oMoVJ1DPc0=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
|
|
@ -148,6 +148,7 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
|
|||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
|
|
|
|||
Loading…
Reference in New Issue