Store rework. Direct query for retrieval now. Fixes #2
parent
01a6b678ab
commit
4e120d0066
1
.env
1
.env
|
@ -1,2 +1,3 @@
|
||||||
DBPATH=/tmp/badger
|
DBPATH=/tmp/badger
|
||||||
NATS=nats://localhost:4222
|
NATS=nats://localhost:4222
|
||||||
|
LISTEN=:80
|
||||||
|
|
99
README.md
99
README.md
|
@ -1,6 +1,8 @@
|
||||||
# backend-store
|
# backend-store
|
||||||
|
|
||||||
Single Badger store to serve bite, transcription and any others. Is kinda bite-centric, so required values revolve around a Bite. Transacts through NATS.
|
Single Badger store to serve bite, transcription and any others. Is kinda bite-centric, so required values revolve around a Bite. Receives stores through [NATS](https://nats.io) while data is retrieved via a http api.
|
||||||
|
|
||||||
|
**Relies on being behind a traefik instance forwarding auth to backend-auth for authentication**
|
||||||
|
|
||||||
## Environment Variables
|
## Environment Variables
|
||||||
|
|
||||||
|
@ -10,28 +12,97 @@ Supply environment variables by either exporting them or editing ```.env```.
|
||||||
| ---- | ----------- | ------- |
|
| ---- | ----------- | ------- |
|
||||||
| DBPATH | Path to store badger files in. Please make sure it exists. | /tmp/badger |
|
| DBPATH | Path to store badger files in. Please make sure it exists. | /tmp/badger |
|
||||||
| NATS | Host and port of nats | nats://localhost:4222 |
|
| NATS | Host and port of nats | nats://localhost:4222 |
|
||||||
|
| LISTEN | Host and port to listen on | :80 |
|
||||||
|
|
||||||
## Key format
|
## Key format
|
||||||
|
|
||||||
Takes in three variables: ```type```, ```key``` and ```start```. Type is the type of data to be inserted, e.g. ```bite```, ```bite_user``` or ```transcription```. Key could be some secret passphrase declaring you the Raj of British India for all I know. Start is the Epoch timestamp of the start of the Bite.
|
Takes in three variables: ```type```, ```key``` and ```start```. Type is the type of data to be inserted, e.g. ```bite``` or ```transcription```. Key is the id of the conversation the bite was said in. Start is the Epoch timestamp of the start of the Bite.
|
||||||
|
|
||||||
## NATS
|
## NATS
|
||||||
|
|
||||||
Refer to protobuf definitions in ```backend-protobuf```.
|
Refer to protobuf definitions in ```backend-protobuf```.
|
||||||
|
|
||||||
| Name | What you do | Accepted Protobuf | Protobuf redundant fields | Response Protobuf | Response empty fields |
|
| Name | What you do | Accepted Protobuf |
|
||||||
| ---- | ----------- | ----------------- | ------------------------- | ----------------- | --------------------- |
|
| ---- | ----------- | ----------------- |
|
||||||
| new_store | Publish to | Store | - | - | - |
|
| store | Publish to | Store |
|
||||||
| request_store | Request | DataRequest | - | Response | client |
|
|
||||||
| scan_store | Request | ScanRequest | - | Response | client |
|
|
||||||
|
|
||||||
|
### store
|
||||||
|
|
||||||
### new_store
|
Succeeds or fails quietly, just logging errors.
|
||||||
|
|
||||||
Pushes the results of its operation to ```backend-subscribe```.
|
## API
|
||||||
|
|
||||||
| Code | Message | Description |
|
| Contents |
|
||||||
| ---- | ------- | ----------- |
|
| -------- |
|
||||||
| 200 | Inserted bite's key | Store operation was successful |
|
| Scan Store |
|
||||||
| 400 | 400 Bad Request | Key could not be marshalled properly |
|
| Get Store |
|
||||||
| 500 | 500 Internal Server Error | Error storing the bite in badger |
|
|
||||||
|
### Scan Store
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /:type/:key/scan
|
||||||
|
```
|
||||||
|
|
||||||
|
Get a list of start times that one can use to query individual bites.
|
||||||
|
|
||||||
|
#### Params
|
||||||
|
|
||||||
|
| Name | Type | Description |
|
||||||
|
| ---- | ---- | ----------- |
|
||||||
|
| type | String | Type of store to query. I.e. `transcription` or `bite`. |
|
||||||
|
| key | String | Conversation ID of bite to query. |
|
||||||
|
|
||||||
|
#### Querystring
|
||||||
|
|
||||||
|
| Name | Type | Description |
|
||||||
|
| ---- | ---- | ----------- |
|
||||||
|
| from | Epoch timestamp | Time to start scanning from. |
|
||||||
|
| to | Epoch timestamp | Time to stop scanning at. |
|
||||||
|
|
||||||
|
#### Success (200 OK)
|
||||||
|
|
||||||
|
All numbers are Unix epoch timestamps. `starts` goes on for as long as it needs to.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"previous": 0,
|
||||||
|
"starts: [0, 0, 0],
|
||||||
|
"next": 0
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Errors
|
||||||
|
|
||||||
|
| Code | Description |
|
||||||
|
| ---- | ----------- |
|
||||||
|
| 400 | From/to are not unix epoch/Error marshalling key from params. |
|
||||||
|
| 500 | Error scanning badger store. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Get Store
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /:type/:key/start/:start
|
||||||
|
```
|
||||||
|
|
||||||
|
Get the bite at a specific timestamp.
|
||||||
|
|
||||||
|
#### Params
|
||||||
|
|
||||||
|
| Name | Type | Description |
|
||||||
|
| ---- | ---- | ----------- |
|
||||||
|
| type | String | Type of store to query. I.e. `transcription` or `bite`. |
|
||||||
|
| key | String | Conversation ID of bite to query. |
|
||||||
|
| start | Epoch timestamp | Timestamp of the bite. |
|
||||||
|
|
||||||
|
#### Success (200 OK)
|
||||||
|
|
||||||
|
Raw data of the bite.
|
||||||
|
|
||||||
|
#### Errors
|
||||||
|
|
||||||
|
| Code | Description |
|
||||||
|
| ---- | ----------- |
|
||||||
|
| 400 | Error marshalling key from params/`start` was not a valid Epoch timestamp. |
|
||||||
|
| 500 | Error retrieving bite from store. |
|
||||||
|
|
1
go.mod
1
go.mod
|
@ -6,6 +6,7 @@ require (
|
||||||
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f // indirect
|
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f // indirect
|
||||||
github.com/golang/protobuf v1.2.1-0.20190205222052-c823c79ea157
|
github.com/golang/protobuf v1.2.1-0.20190205222052-c823c79ea157
|
||||||
github.com/joho/godotenv v1.3.0
|
github.com/joho/godotenv v1.3.0
|
||||||
|
github.com/julienschmidt/httprouter v1.2.0
|
||||||
github.com/nats-io/gnatsd v1.4.1 // indirect
|
github.com/nats-io/gnatsd v1.4.1 // indirect
|
||||||
github.com/nats-io/go-nats v1.7.0
|
github.com/nats-io/go-nats v1.7.0
|
||||||
github.com/nats-io/nkeys v0.0.2 // indirect
|
github.com/nats-io/nkeys v0.0.2 // indirect
|
||||||
|
|
2
go.sum
2
go.sum
|
@ -10,6 +10,8 @@ github.com/golang/protobuf v1.2.1-0.20190205222052-c823c79ea157 h1:SdQMHsZ18/XZC
|
||||||
github.com/golang/protobuf v1.2.1-0.20190205222052-c823c79ea157/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
github.com/golang/protobuf v1.2.1-0.20190205222052-c823c79ea157/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
|
||||||
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
|
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
|
||||||
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
|
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
|
||||||
|
github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
|
||||||
|
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||||
github.com/nats-io/gnatsd v1.4.1 h1:RconcfDeWpKCD6QIIwiVFcvForlXpWeJP7i5/lDLy44=
|
github.com/nats-io/gnatsd v1.4.1 h1:RconcfDeWpKCD6QIIwiVFcvForlXpWeJP7i5/lDLy44=
|
||||||
github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ=
|
github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ=
|
||||||
github.com/nats-io/go-nats v1.7.0 h1:oQOfHcLr8hb43QG8yeVyY2jtarIaTjOv41CGdF3tTvQ=
|
github.com/nats-io/go-nats v1.7.0 h1:oQOfHcLr8hb43QG8yeVyY2jtarIaTjOv41CGdF3tTvQ=
|
||||||
|
|
185
main.go
185
main.go
|
@ -5,6 +5,7 @@ import (
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
. "store/backend-protobuf/go"
|
. "store/backend-protobuf/go"
|
||||||
|
|
||||||
|
@ -12,8 +13,10 @@ import (
|
||||||
"github.com/dgraph-io/badger"
|
"github.com/dgraph-io/badger"
|
||||||
"github.com/nats-io/go-nats"
|
"github.com/nats-io/go-nats"
|
||||||
"github.com/golang/protobuf/proto"
|
"github.com/golang/protobuf/proto"
|
||||||
|
"github.com/julienschmidt/httprouter"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var listen string
|
||||||
var dbPath string
|
var dbPath string
|
||||||
var natsHost string
|
var natsHost string
|
||||||
|
|
||||||
|
@ -28,6 +31,7 @@ func main() {
|
||||||
}
|
}
|
||||||
dbPath = os.Getenv("DBPATH")
|
dbPath = os.Getenv("DBPATH")
|
||||||
natsHost = os.Getenv("NATS")
|
natsHost = os.Getenv("NATS")
|
||||||
|
listen = os.Getenv("LISTEN")
|
||||||
|
|
||||||
// Open badger
|
// Open badger
|
||||||
log.Printf("starting badger at %s", dbPath)
|
log.Printf("starting badger at %s", dbPath)
|
||||||
|
@ -45,34 +49,29 @@ func main() {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
nc.Subscribe("new_store", NewStore)
|
nc.Subscribe("store", NewStore)
|
||||||
|
|
||||||
nc.Subscribe("request_store", RequestStore)
|
|
||||||
nc.Subscribe("scan_store", ScanStore)
|
|
||||||
defer nc.Close()
|
defer nc.Close()
|
||||||
|
|
||||||
select { } // Wait forever
|
// Routes
|
||||||
|
router := httprouter.New()
|
||||||
|
router.GET("/:type/:key/scan", ScanStore)
|
||||||
|
router.GET("/:type/:key/start/:start", GetStore)
|
||||||
|
|
||||||
|
// Start server
|
||||||
|
log.Printf("starting server on %s", listen)
|
||||||
|
log.Fatal(http.ListenAndServe(listen, router))
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewStore(m *nats.Msg) {
|
func NewStore(m *nats.Msg) {
|
||||||
storeRequest := Store{}
|
storeRequest := Store{}
|
||||||
if err := proto.Unmarshal(m.Data, &storeRequest); err != nil {
|
if err := proto.Unmarshal(m.Data, &storeRequest); err != nil {
|
||||||
log.Println(err) // Fail quietly since protobuf data is needed torespond
|
log.Println(err) // Just log errors
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
key, err := MarshalKey(storeRequest.Type, storeRequest.Bite.Key, storeRequest.Bite.Start)
|
key, err := MarshalKey(storeRequest.Type, storeRequest.Bite.Key, storeRequest.Bite.Start)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println(err)
|
log.Println(err)
|
||||||
errRes := Response {
|
|
||||||
Code: 400,
|
|
||||||
Message: []byte(http.StatusText(http.StatusBadRequest)),
|
|
||||||
Client: storeRequest.Bite.Client,
|
|
||||||
}
|
|
||||||
errResBytes, errResErr := proto.Marshal(&errRes)
|
|
||||||
if errResErr == nil {
|
|
||||||
nc.Publish("res", errResBytes)
|
|
||||||
}
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -81,69 +80,52 @@ func NewStore(m *nats.Msg) {
|
||||||
err := txn.Set(key, storeRequest.Bite.Data)
|
err := txn.Set(key, storeRequest.Bite.Data)
|
||||||
return err
|
return err
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Println(err)
|
log.Println(err)
|
||||||
errRes := Response {
|
|
||||||
Code: 500,
|
|
||||||
Message: []byte(http.StatusText(http.StatusInternalServerError)),
|
|
||||||
Client: storeRequest.Bite.Client,
|
|
||||||
}
|
|
||||||
errResBytes, errResErr := proto.Marshal(&errRes)
|
|
||||||
if errResErr == nil {
|
|
||||||
nc.Publish("res", errResBytes)
|
|
||||||
}
|
|
||||||
return
|
return
|
||||||
} else {
|
|
||||||
res := Response {
|
|
||||||
Code: 200,
|
|
||||||
Message: []byte(key),
|
|
||||||
Client: storeRequest.Bite.Client,
|
|
||||||
}
|
|
||||||
resBytes, resErr := proto.Marshal(&res)
|
|
||||||
if resErr == nil {
|
|
||||||
nc.Publish("res", resBytes)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func RequestStore(m *nats.Msg) {
|
func ParseStartString(start string) (uint64, error) {
|
||||||
req := DataRequest{}
|
return strconv.ParseUint(start, 10, 64)
|
||||||
if err := proto.Unmarshal(m.Data, &req); err != nil {
|
}
|
||||||
log.Println(err)
|
|
||||||
|
func GetStore(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
|
||||||
|
// Get params
|
||||||
|
storeType := p.ByName("type")
|
||||||
|
key := p.ByName("key")
|
||||||
|
|
||||||
|
start, err := ParseStartString(p.ByName("start"))
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
key, err := MarshalKey(req.Type, req.Key, req.Start)
|
storeKey, err := MarshalKey(storeType, key, start)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
err = db.View(func(txn *badger.Txn) error {
|
err = db.View(func(txn *badger.Txn) error {
|
||||||
item, err := txn.Get(key)
|
item, err := txn.Get(storeKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
value, err := item.Value()
|
value, err := item.Value()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
res := Response {
|
w.Write(value)
|
||||||
Code: 200,
|
|
||||||
Message: value,
|
|
||||||
}
|
|
||||||
resBytes, err := proto.Marshal(&res)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
nc.Publish(m.Reply, resBytes)
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
res := ReplyError(err.Error(), 400)
|
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||||
nc.Publish(m.Reply, res)
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type BitesList struct {
|
type BitesList struct {
|
||||||
|
@ -152,29 +134,37 @@ type BitesList struct {
|
||||||
Next uint64 `json:"next"` // One bite after starts. Hint for how many steps the client can skip
|
Next uint64 `json:"next"` // One bite after starts. Hint for how many steps the client can skip
|
||||||
}
|
}
|
||||||
|
|
||||||
func ScanStore(m *nats.Msg) {
|
func ScanStore(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
|
||||||
req := ScanRequest {}
|
// Get params
|
||||||
if err := proto.Unmarshal(m.Data, &req); err != nil {
|
storeType := p.ByName("type")
|
||||||
log.Println(err)
|
key := p.ByName("key")
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix, err := MarshalKeyPrefix(req.Type, req.Key)
|
|
||||||
|
|
||||||
|
// Get querystring values
|
||||||
|
from, err := ParseStartString(r.FormValue("from"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
res := ReplyError(http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||||
nc.Publish(m.Reply, res)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
fromKey, err := MarshalKey(req.Type, req.Key, req.From)
|
to, err := ParseStartString(r.FormValue("to"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
res := ReplyError(http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||||
nc.Publish(m.Reply, res)
|
return
|
||||||
return
|
}
|
||||||
}
|
|
||||||
|
|
||||||
bitesList := BitesList {}
|
prefix, err := MarshalKeyPrefix(storeType, key)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fromKey, err := MarshalKey(storeType, key, from)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
bitesList := BitesList{}
|
||||||
|
|
||||||
err = db.View(func(txn *badger.Txn) error {
|
err = db.View(func(txn *badger.Txn) error {
|
||||||
opts := badger.DefaultIteratorOptions
|
opts := badger.DefaultIteratorOptions
|
||||||
|
@ -202,12 +192,11 @@ func ScanStore(m *nats.Msg) {
|
||||||
bitesList.Previous = start
|
bitesList.Previous = start
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
res := ReplyError(err.Error(), http.StatusBadRequest)
|
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||||
nc.Publish(m.Reply, res)
|
return
|
||||||
return
|
}
|
||||||
}
|
|
||||||
|
|
||||||
err = db.View(func(txn *badger.Txn) error {
|
err = db.View(func(txn *badger.Txn) error {
|
||||||
opts := badger.DefaultIteratorOptions
|
opts := badger.DefaultIteratorOptions
|
||||||
|
@ -223,7 +212,7 @@ func ScanStore(m *nats.Msg) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if start > req.To {
|
if start > to {
|
||||||
// A key was found that is greater than to
|
// A key was found that is greater than to
|
||||||
// Save that as next
|
// Save that as next
|
||||||
bitesList.Next = start
|
bitesList.Next = start
|
||||||
|
@ -235,27 +224,13 @@ func ScanStore(m *nats.Msg) {
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
res := ReplyError(err.Error(), http.StatusBadRequest)
|
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||||
nc.Publish(m.Reply, res)
|
return
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
jsonString, err := json.Marshal(&bitesList)
|
|
||||||
res := Response {
|
|
||||||
Code: 200,
|
|
||||||
Message: []byte(jsonString),
|
|
||||||
}
|
}
|
||||||
resBytes, _ := proto.Marshal(&res)
|
|
||||||
nc.Publish(m.Reply, resBytes)
|
// Respond
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
json.NewEncoder(w).Encode(bitesList)
|
||||||
}
|
}
|
||||||
|
|
||||||
func ReplyError(msg string, code uint32) []byte {
|
|
||||||
res := Response {
|
|
||||||
Code: code,
|
|
||||||
Message: []byte(msg),
|
|
||||||
}
|
|
||||||
resBytes, _ := proto.Marshal(&res)
|
|
||||||
|
|
||||||
return resBytes
|
|
||||||
}
|
|
||||||
|
|
Loading…
Reference in New Issue