Initial commit

2019-02-08 07:30:21 +08:00 · 2019-02-08 07:30:21 +08:00 · 0e172cdcbd
parent 989ba5db6b
commit 0e172cdcbd
4 changed files with 474 additions and 1 deletions
--- a/README.md
+++ b/README.md
@ -1,3 +1,39 @@
 # backend-bite

-Beep backend handling of audio bites.
+Beep backend handling of audio bites. Chopped up words spoken are uploaded into bite to be stored in a scan-able key-value store.  Subscribes to ```new_bite``` and ```new_bite_user``` events from a NATS publisher.
+
+## Quickstart
+
+```
+go build && ./backend-bite
+```
+
+## Flags
+
+Flags are supplied to the compiled go program in the form ```-flag=stuff```.
+
+| Flag | Description | Default |
+| ---- | ----------- | ------- |
+| listen | Port number to listen on | 8080 |
+| dbpath | File path to store DB data | /tmp/badger |
+| nats | URL of NATS | nats://localhost:4222 |
+
+## API
+
+### Scan Bites
+
+```
+GET /conversation/:key/scan
+```
+
+### Get Bite
+
+```
+GET /conversation/:key/start/:start
+```
+
+### Get Bite User
+
+```
+GET /conversation/:key/start/:start/user
+```
--- a/backend-bite.exe
+++ b/backend-bite.exe
--- a/bite.pb.go
+++ b/bite.pb.go
@ -0,0 +1,93 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: bite.proto
+
+package main
+
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+type Bite struct {
+	Key                  string   `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+	Start                uint64   `protobuf:"varint,2,opt,name=start,proto3" json:"start,omitempty"`
+	Data                 []byte   `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *Bite) Reset()         { *m = Bite{} }
+func (m *Bite) String() string { return proto.CompactTextString(m) }
+func (*Bite) ProtoMessage()    {}
+func (*Bite) Descriptor() ([]byte, []int) {
+	return fileDescriptor_e1ec993646b17549, []int{0}
+}
+
+func (m *Bite) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Bite.Unmarshal(m, b)
+}
+func (m *Bite) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Bite.Marshal(b, m, deterministic)
+}
+func (m *Bite) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Bite.Merge(m, src)
+}
+func (m *Bite) XXX_Size() int {
+	return xxx_messageInfo_Bite.Size(m)
+}
+func (m *Bite) XXX_DiscardUnknown() {
+	xxx_messageInfo_Bite.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Bite proto.InternalMessageInfo
+
+func (m *Bite) GetKey() string {
+	if m != nil {
+		return m.Key
+	}
+	return ""
+}
+
+func (m *Bite) GetStart() uint64 {
+	if m != nil {
+		return m.Start
+	}
+	return 0
+}
+
+func (m *Bite) GetData() []byte {
+	if m != nil {
+		return m.Data
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*Bite)(nil), "main.Bite")
+}
+
+func init() { proto.RegisterFile("bite.proto", fileDescriptor_e1ec993646b17549) }
+
+var fileDescriptor_e1ec993646b17549 = []byte{
+	// 105 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4a, 0xca, 0x2c, 0x49,
+	0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0xc9, 0x4d, 0xcc, 0xcc, 0x53, 0x72, 0xe2, 0x62,
+	0x71, 0xca, 0x2c, 0x49, 0x15, 0x12, 0xe0, 0x62, 0xce, 0x4e, 0xad, 0x94, 0x60, 0x54, 0x60, 0xd4,
+	0xe0, 0x0c, 0x02, 0x31, 0x85, 0x44, 0xb8, 0x58, 0x8b, 0x4b, 0x12, 0x8b, 0x4a, 0x24, 0x98, 0x14,
+	0x18, 0x35, 0x58, 0x82, 0x20, 0x1c, 0x21, 0x21, 0x2e, 0x96, 0x94, 0xc4, 0x92, 0x44, 0x09, 0x66,
+	0x05, 0x46, 0x0d, 0x9e, 0x20, 0x30, 0x3b, 0x89, 0x0d, 0x6c, 0xa0, 0x31, 0x20, 0x00, 0x00, 0xff,
+	0xff, 0x28, 0xbd, 0x24, 0x95, 0x5e, 0x00, 0x00, 0x00,
+}
--- a/main.go
+++ b/main.go
@ -0,0 +1,344 @@
+package main
+
+import (
+  "bytes"
+  "encoding/binary"
+  "encoding/json"
+  "errors"
+  "flag"
+  "net/http"
+  "log"
+  "regexp"
+  "strconv"
+
+  "github.com/dgraph-io/badger"
+	"github.com/julienschmidt/httprouter"
+  "github.com/nats-io/go-nats"
+  "github.com/golang/protobuf/proto"
+)
+
+var listen string
+var dbPath string
+var natsHost string
+
+var db *badger.DB
+
+func main() {
+  // Parse flags
+  flag.StringVar(&listen, "listen", ":8080", "host and port to listen on")
+  flag.StringVar(&dbPath, "dbpath", "/tmp/badger", "path to store data")
+  flag.StringVar(&natsHost, "nats", "nats://localhost:4222", "host and port of NATS")
+	flag.Parse()
+
+  // Open badger
+	log.Printf("starting badger at %s", dbPath)
+	opts := badger.DefaultOptions
+	opts.Dir = dbPath
+	opts.ValueDir = dbPath
+	var err error
+	db, err = badger.Open(opts)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+
+  // NATS client
+  nc, _ := nats.Connect(natsHost);
+  nc.Subscribe("new_bite", NewBite);
+  nc.Subscribe("new_bite_user", NewBiteUser);
+
+  // Routes
+	router := httprouter.New()
+	router.GET("/conversation/:key/scan", ScanBites) // Scanning
+  router.GET("/conversation/:key/start/:start", GetBite) // GET bites
+  router.GET("/conversation/:key/start/:start/user", GetBiteUser) // GET bite_users
+
+  // Start server
+  log.Printf("starting server on %s", listen)
+	log.Fatal(http.ListenAndServe(listen, router))
+}
+
+// Marshal keys
+func validObj(obj string) bool {
+	return obj == "bite" || obj == "user"
+}
+
+// TODO: ensure security of regexp
+var validConversationRegexp = regexp.MustCompile(`^[a-zA-Z0-9-]+$`)
+
+func validConversation(conversation string) bool {
+	return validConversationRegexp.MatchString(conversation)
+}
+
+const conversationSeprator = '@'
+const objSeprator = '+'
+
+func MarshalKey(obj, conversation string, start uint64) ([]byte, error) {
+	prefixBytes, err := MarshalKeyPrefix(obj, conversation)
+	if err != nil {
+		return nil, err
+	}
+
+	startBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(startBytes, start)
+
+	return append(prefixBytes, startBytes...), nil
+}
+
+func MarshalKeyPrefix(obj, conversation string) ([]byte, error) {
+	if !validObj(obj) || !validConversation(conversation) {
+		return nil, errors.New("main: FormatKey: bad obj or conversation")
+	}
+	return []byte(obj + string(objSeprator) + conversation + string(conversationSeprator)), nil
+}
+
+func ExtractKey(b []byte) (string, string, uint64, error) {
+	startStart := bytes.LastIndexByte(b, conversationSeprator) + 1
+	if startStart < 0 {
+		return "", "", 0, ExtractKeyParseError
+	}
+	startBytes := b[startStart:]
+
+	convStart := bytes.LastIndexByte(b[:startStart-1], objSeprator) + 1
+	if convStart < 0 {
+		return "", "", 0, ExtractKeyParseError
+	}
+	convBytes := b[convStart : startStart-1]
+
+	objStart := 0
+	if objStart < 0 {
+		return "", "", 0, ExtractKeyParseError
+	}
+	objBytes := b[objStart : convStart-1]
+
+	obj := string(objBytes)
+	conv := string(convBytes)
+	start := binary.BigEndian.Uint64(startBytes)
+
+	return obj, conv, start, nil
+}
+
+var ExtractKeyParseError = errors.New("ExtractKey: parse error, possibly because seprator was not found")
+
+func ParseStartString(start string) (uint64, error) {
+	return strconv.ParseUint(start, 10, 64)
+}
+
+// Sub handlers
+// m.data = Bite protobuf
+func NewBite(m *nats.Msg) {
+  bite := Bite{}
+  if err := proto.Unmarshal(m.Data, &bite); err != nil {
+    log.Println(err)
+    return
+  }
+
+  key, err := MarshalKey("bite", bite.Key, bite.Start)
+  if err != nil {
+    log.Println(err)
+    return
+  }
+
+  err = db.Update(func(txn *badger.Txn) error {
+		// TODO: prevent overwriting existing
+		err := txn.Set(key, bite.Data)
+		return err
+	})
+
+  if err != nil {
+    log.Println(err)
+    return
+  }
+}
+
+func NewBiteUser(m *nats.Msg) {
+  bite := Bite{}
+  if err := proto.Unmarshal(m.Data, &bite); err != nil {
+    log.Println(err)
+    return
+  }
+
+  key, err := MarshalKey("user", bite.Key, bite.Start)
+  if err != nil {
+    log.Println(err)
+    return
+  }
+
+  err = db.Update(func(txn *badger.Txn) error {
+    // TODO: prevent overwriting existing
+    err := txn.Set(key, bite.Data)
+    return err
+  })
+
+  if err != nil {
+    log.Println(err)
+    return
+  }
+}
+
+// rOUTE HANDLERS
+
+type BitesList struct {
+	Previous uint64   `json:"previous"` // One bite before starts. Hint for how many steps the client can skip
+	Starts   []uint64 `json:"starts"`
+	Next     uint64   `json:"next"` // One bite after starts. Hint for how many steps the client can skip
+}
+
+func ScanBites(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+  prefix, err := MarshalKeyPrefix("bite", p.ByName("key"))
+	if err != nil {
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+
+	from, err := ParseStartString(r.FormValue("from"))
+	if err != nil {
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+	to, err := ParseStartString(r.FormValue("to"))
+	if err != nil {
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+
+	fromKey, err := MarshalKey("bite", p.ByName("key"), from)
+	if err != nil {
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+
+	bitesList := BitesList{}
+
+	err = db.View(func(txn *badger.Txn) error {
+		opts := badger.DefaultIteratorOptions
+		opts.PrefetchValues = false
+		opts.Reverse = true
+		it := txn.NewIterator(opts)
+		defer it.Close()
+
+		// Fetch previous key
+		it.Seek(fromKey)
+		if it.ValidForPrefix(fromKey) {
+			// Lazy check to compare key == seeked key
+			it.Next()
+		}
+		if !it.ValidForPrefix(prefix) {
+			return nil
+		}
+		item := it.Item()
+		key := item.Key()
+
+		_, _, start, err := ExtractKey(key)
+		if err != nil {
+			return nil
+		}
+		bitesList.Previous = start
+
+		return nil
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	err = db.View(func(txn *badger.Txn) error {
+		opts := badger.DefaultIteratorOptions
+		opts.PrefetchValues = false
+		it := txn.NewIterator(opts)
+		defer it.Close()
+
+		for it.Seek(fromKey); it.ValidForPrefix(prefix); it.Next() {
+			item := it.Item()
+			key := item.Key()
+
+			_, _, start, err := ExtractKey(key)
+			if err != nil {
+				continue
+			}
+			if start > to {
+				// A key was found that is greater than to
+				// Save that as next
+				bitesList.Next = start
+				break
+			}
+
+			bitesList.Starts = append(bitesList.Starts, start)
+		}
+
+		return nil
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(bitesList)
+}
+
+func GetBite(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+	w.Header().Add("Content-Type", "audio/wav")
+	start, err := ParseStartString(p.ByName("start"))
+	if err != nil {
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+	key, err := MarshalKey("bite", p.ByName("key"), start)
+	if err != nil {
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+
+	err = db.View(func(txn *badger.Txn) error {
+		item, err := txn.Get(key)
+		if err != nil {
+			return err
+		}
+    err = item.Value(func(value []byte) error {
+      w.Write(value)
+      return nil
+    })
+    if err != nil {
+      return err
+    }
+    return nil
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+}
+
+func GetBiteUser(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+	w.Header().Add("Content-Type", "text/plain")
+	start, err := ParseStartString(p.ByName("start"))
+	if err != nil {
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+	key, err := MarshalKey("user", p.ByName("key"), start)
+	if err != nil {
+		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
+		return
+	}
+
+	err = db.View(func(txn *badger.Txn) error {
+		item, err := txn.Get(key)
+		if err != nil {
+			return err
+		}
+    err = item.Value(func(value []byte) error {
+      w.Write(value)
+      return nil
+    })
+    if err != nil {
+      return err
+    }
+    return nil
+	})
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+}