diff --git a/.env b/.env index 4715d14..baaeaa5 100644 --- a/.env +++ b/.env @@ -1,2 +1,3 @@ DBPATH=/tmp/badger NATS=nats://localhost:4222 +LISTEN=:80 diff --git a/README.md b/README.md index d325d7a..2595e90 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # backend-store -Single Badger store to serve bite, transcription and any others. Is kinda bite-centric, so required values revolve around a Bite. Transacts through NATS. +Single Badger store to serve bite, transcription and any others. Is kinda bite-centric, so required values revolve around a Bite. Receives stores through [NATS](https://nats.io) while data is retrieved via a http api. + +**Relies on being behind a traefik instance forwarding auth to backend-auth for authentication** ## Environment Variables @@ -10,28 +12,97 @@ Supply environment variables by either exporting them or editing ```.env```. | ---- | ----------- | ------- | | DBPATH | Path to store badger files in. Please make sure it exists. | /tmp/badger | | NATS | Host and port of nats | nats://localhost:4222 | +| LISTEN | Host and port to listen on | :80 | ## Key format -Takes in three variables: ```type```, ```key``` and ```start```. Type is the type of data to be inserted, e.g. ```bite```, ```bite_user``` or ```transcription```. Key could be some secret passphrase declaring you the Raj of British India for all I know. Start is the Epoch timestamp of the start of the Bite. +Takes in three variables: ```type```, ```key``` and ```start```. Type is the type of data to be inserted, e.g. ```bite``` or ```transcription```. Key is the id of the conversation the bite was said in. Start is the Epoch timestamp of the start of the Bite. ## NATS Refer to protobuf definitions in ```backend-protobuf```. -| Name | What you do | Accepted Protobuf | Protobuf redundant fields | Response Protobuf | Response empty fields | -| ---- | ----------- | ----------------- | ------------------------- | ----------------- | --------------------- | -| new_store | Publish to | Store | - | - | - | -| request_store | Request | DataRequest | - | Response | client | -| scan_store | Request | ScanRequest | - | Response | client | +| Name | What you do | Accepted Protobuf | +| ---- | ----------- | ----------------- | +| store | Publish to | Store | +### store -### new_store +Succeeds or fails quietly, just logging errors. -Pushes the results of its operation to ```backend-subscribe```. +## API -| Code | Message | Description | -| ---- | ------- | ----------- | -| 200 | Inserted bite's key | Store operation was successful | -| 400 | 400 Bad Request | Key could not be marshalled properly | -| 500 | 500 Internal Server Error | Error storing the bite in badger | +| Contents | +| -------- | +| Scan Store | +| Get Store | + +### Scan Store + +``` +GET /:type/:key/scan +``` + +Get a list of start times that one can use to query individual bites. + +#### Params + +| Name | Type | Description | +| ---- | ---- | ----------- | +| type | String | Type of store to query. I.e. `transcription` or `bite`. | +| key | String | Conversation ID of bite to query. | + +#### Querystring + +| Name | Type | Description | +| ---- | ---- | ----------- | +| from | Epoch timestamp | Time to start scanning from. | +| to | Epoch timestamp | Time to stop scanning at. | + +#### Success (200 OK) + +All numbers are Unix epoch timestamps. `starts` goes on for as long as it needs to. + +```json +{ + "previous": 0, + "starts: [0, 0, 0], + "next": 0 +} +``` + +#### Errors + +| Code | Description | +| ---- | ----------- | +| 400 | From/to are not unix epoch/Error marshalling key from params. | +| 500 | Error scanning badger store. | + +--- + +### Get Store + +``` +GET /:type/:key/start/:start +``` + +Get the bite at a specific timestamp. + +#### Params + +| Name | Type | Description | +| ---- | ---- | ----------- | +| type | String | Type of store to query. I.e. `transcription` or `bite`. | +| key | String | Conversation ID of bite to query. | +| start | Epoch timestamp | Timestamp of the bite. | + +#### Success (200 OK) + +Raw data of the bite. + +#### Errors + +| Code | Description | +| ---- | ----------- | +| 400 | Error marshalling key from params/`start` was not a valid Epoch timestamp. | +| 500 | Error retrieving bite from store. | diff --git a/go.mod b/go.mod index 8e9fba2..2028ea7 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f // indirect github.com/golang/protobuf v1.2.1-0.20190205222052-c823c79ea157 github.com/joho/godotenv v1.3.0 + github.com/julienschmidt/httprouter v1.2.0 github.com/nats-io/gnatsd v1.4.1 // indirect github.com/nats-io/go-nats v1.7.0 github.com/nats-io/nkeys v0.0.2 // indirect diff --git a/go.sum b/go.sum index 5f943d4..ab1b38a 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/golang/protobuf v1.2.1-0.20190205222052-c823c79ea157 h1:SdQMHsZ18/XZC github.com/golang/protobuf v1.2.1-0.20190205222052-c823c79ea157/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/nats-io/gnatsd v1.4.1 h1:RconcfDeWpKCD6QIIwiVFcvForlXpWeJP7i5/lDLy44= github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= github.com/nats-io/go-nats v1.7.0 h1:oQOfHcLr8hb43QG8yeVyY2jtarIaTjOv41CGdF3tTvQ= diff --git a/main.go b/main.go index bc85c9c..737129e 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "log" "net/http" "os" + "strconv" . "store/backend-protobuf/go" @@ -12,8 +13,10 @@ import ( "github.com/dgraph-io/badger" "github.com/nats-io/go-nats" "github.com/golang/protobuf/proto" + "github.com/julienschmidt/httprouter" ) +var listen string var dbPath string var natsHost string @@ -28,6 +31,7 @@ func main() { } dbPath = os.Getenv("DBPATH") natsHost = os.Getenv("NATS") + listen = os.Getenv("LISTEN") // Open badger log.Printf("starting badger at %s", dbPath) @@ -45,34 +49,29 @@ func main() { if err != nil { log.Fatal(err) } - nc.Subscribe("new_store", NewStore) - - nc.Subscribe("request_store", RequestStore) - nc.Subscribe("scan_store", ScanStore) + nc.Subscribe("store", NewStore) defer nc.Close() - select { } // Wait forever + // Routes + router := httprouter.New() + router.GET("/:type/:key/scan", ScanStore) + router.GET("/:type/:key/start/:start", GetStore) + + // Start server + log.Printf("starting server on %s", listen) + log.Fatal(http.ListenAndServe(listen, router)) } func NewStore(m *nats.Msg) { storeRequest := Store{} if err := proto.Unmarshal(m.Data, &storeRequest); err != nil { - log.Println(err) // Fail quietly since protobuf data is needed torespond + log.Println(err) // Just log errors return } key, err := MarshalKey(storeRequest.Type, storeRequest.Bite.Key, storeRequest.Bite.Start) if err != nil { log.Println(err) - errRes := Response { - Code: 400, - Message: []byte(http.StatusText(http.StatusBadRequest)), - Client: storeRequest.Bite.Client, - } - errResBytes, errResErr := proto.Marshal(&errRes) - if errResErr == nil { - nc.Publish("res", errResBytes) - } return } @@ -81,69 +80,52 @@ func NewStore(m *nats.Msg) { err := txn.Set(key, storeRequest.Bite.Data) return err }) - if err != nil { log.Println(err) - errRes := Response { - Code: 500, - Message: []byte(http.StatusText(http.StatusInternalServerError)), - Client: storeRequest.Bite.Client, - } - errResBytes, errResErr := proto.Marshal(&errRes) - if errResErr == nil { - nc.Publish("res", errResBytes) - } return - } else { - res := Response { - Code: 200, - Message: []byte(key), - Client: storeRequest.Bite.Client, - } - resBytes, resErr := proto.Marshal(&res) - if resErr == nil { - nc.Publish("res", resBytes) - } } } -func RequestStore(m *nats.Msg) { - req := DataRequest{} - if err := proto.Unmarshal(m.Data, &req); err != nil { - log.Println(err) +func ParseStartString(start string) (uint64, error) { + return strconv.ParseUint(start, 10, 64) +} + +func GetStore(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + // Get params + storeType := p.ByName("type") + key := p.ByName("key") + + start, err := ParseStartString(p.ByName("start")) + if err != nil { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) return } - key, err := MarshalKey(req.Type, req.Key, req.Start) + storeKey, err := MarshalKey(storeType, key, start) + if err != nil { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } err = db.View(func(txn *badger.Txn) error { - item, err := txn.Get(key) - if err != nil { - return err - } + item, err := txn.Get(storeKey) + if err != nil { + return err + } + value, err := item.Value() if err != nil { return err } - res := Response { - Code: 200, - Message: value, - } - resBytes, err := proto.Marshal(&res) - - if err != nil { - return err - } - - nc.Publish(m.Reply, resBytes) + w.Write(value) return nil - }) + }) - if err != nil { - res := ReplyError(err.Error(), 400) - nc.Publish(m.Reply, res) - } + if err != nil { + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } } type BitesList struct { @@ -152,29 +134,37 @@ type BitesList struct { Next uint64 `json:"next"` // One bite after starts. Hint for how many steps the client can skip } -func ScanStore(m *nats.Msg) { - req := ScanRequest {} - if err := proto.Unmarshal(m.Data, &req); err != nil { - log.Println(err) - return - } - - prefix, err := MarshalKeyPrefix(req.Type, req.Key) +func ScanStore(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + // Get params + storeType := p.ByName("type") + key := p.ByName("key") + // Get querystring values + from, err := ParseStartString(r.FormValue("from")) if err != nil { - res := ReplyError(http.StatusText(http.StatusBadRequest), http.StatusBadRequest) - nc.Publish(m.Reply, res) + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) return } - fromKey, err := MarshalKey(req.Type, req.Key, req.From) - if err != nil { - res := ReplyError(http.StatusText(http.StatusBadRequest), http.StatusBadRequest) - nc.Publish(m.Reply, res) - return - } + to, err := ParseStartString(r.FormValue("to")) + if err != nil { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } - bitesList := BitesList {} + prefix, err := MarshalKeyPrefix(storeType, key) + if err != nil { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + + fromKey, err := MarshalKey(storeType, key, from) + if err != nil { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + + bitesList := BitesList{} err = db.View(func(txn *badger.Txn) error { opts := badger.DefaultIteratorOptions @@ -202,12 +192,11 @@ func ScanStore(m *nats.Msg) { bitesList.Previous = start return nil - }) - if err != nil { - res := ReplyError(err.Error(), http.StatusBadRequest) - nc.Publish(m.Reply, res) - return - } + }) + if err != nil { + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } err = db.View(func(txn *badger.Txn) error { opts := badger.DefaultIteratorOptions @@ -223,7 +212,7 @@ func ScanStore(m *nats.Msg) { if err != nil { continue } - if start > req.To { + if start > to { // A key was found that is greater than to // Save that as next bitesList.Next = start @@ -235,27 +224,13 @@ func ScanStore(m *nats.Msg) { return nil }) - if err != nil { - res := ReplyError(err.Error(), http.StatusBadRequest) - nc.Publish(m.Reply, res) - return - } - - jsonString, err := json.Marshal(&bitesList) - res := Response { - Code: 200, - Message: []byte(jsonString), + if err != nil { + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return } - resBytes, _ := proto.Marshal(&res) - nc.Publish(m.Reply, resBytes) + + // Respond + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(bitesList) } -func ReplyError(msg string, code uint32) []byte { - res := Response { - Code: code, - Message: []byte(msg), - } - resBytes, _ := proto.Marshal(&res) - - return resBytes -} diff --git a/store b/store new file mode 100755 index 0000000..2e324da Binary files /dev/null and b/store differ